| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_internvl_chat.py |
| 5 | # -------------------------------------------------------- |
| 6 | # InternVL |
| 7 | # Copyright (c) 2023 OpenGVLab |
| 8 | # Licensed under The MIT License [see LICENSE for details] |
| 9 | # -------------------------------------------------------- |
| 10 | |
| 11 | import numpy.typing as npt |
| 12 | import torch |
| 13 | import torchvision.transforms as T |
| 14 | from PIL import Image |
| 15 | from transformers import ( |
| 16 | BaseVideoProcessor, |
| 17 | BatchFeature, |
| 18 | ImageProcessingMixin, |
| 19 | TensorType, |
| 20 | ) |
| 21 | from transformers.processing_utils import ProcessorMixin |
| 22 | |
| 23 | from vllm.multimodal.image import convert_image_mode |
| 24 | from vllm.multimodal.processing import PromptUpdateDetails, cached_encode |
| 25 | from vllm.tokenizers.hf import HfTokenizer |
| 26 | |
| 27 | IMAGENET_MEAN = (0.485, 0.456, 0.406) |
| 28 | IMAGENET_STD = (0.229, 0.224, 0.225) |
| 29 | |
| 30 | |
| 31 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B |
| 32 | def build_transform(input_size: int): |
| 33 | MEAN, STD = IMAGENET_MEAN, IMAGENET_STD |
| 34 | return T.Compose( |
| 35 | [ |
| 36 | T.Lambda(lambda img: convert_image_mode(img, "RGB")), |
| 37 | T.Resize( |
| 38 | (input_size, input_size), interpolation=T.InterpolationMode.BICUBIC |
| 39 | ), |
| 40 | T.ToTensor(), |
| 41 | T.Normalize(mean=MEAN, std=STD), |
| 42 | ] |
| 43 | ) |
| 44 | |
| 45 | |
| 46 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B |
| 47 | def find_closest_aspect_ratio( |
| 48 | aspect_ratio: float, |
| 49 | target_ratios: list[tuple[int, int]], |
| 50 | *, |
| 51 | width: int, |
| 52 | height: int, |
| 53 | image_size: int, |
| 54 | ) -> tuple[int, int]: |
| 55 | best_ratio_diff = float("inf") |
| 56 | best_ratio = (1, 1) |
| 57 | area = width * height |
| 58 | for ratio in target_ratios: |
| 59 | target_aspect_ratio = ratio[0] / ratio[1] |
| 60 | ratio_diff = abs(aspect_ratio - target_aspect_ratio) |
| 61 | if ratio_diff < best_ratio_diff: |
| 62 | best_ratio_diff = ratio_diff |
| 63 | best_ratio = ratio |
| 64 | elif ratio_diff == best_ratio_diff: |
| 65 | if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: |
| 66 | best_ratio = ratio |
| 67 | return best_ratio |
| 68 | |
| 69 | |
| 70 | def resolve_internvl_min_max_num( |
| 71 | *, |
| 72 | min_dynamic_patch: int, |
| 73 | max_dynamic_patch: int, |
| 74 | dynamic_image_size: bool, |
| 75 | use_thumbnail: bool, |
| 76 | ) -> tuple[int, int]: |
| 77 | min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1 |
| 78 | max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1 |
| 79 | |
| 80 | if use_thumbnail and max_dynamic_patch != 1: |
| 81 | max_dynamic_patch += 1 |
| 82 | |
| 83 | return min_dynamic_patch, max_dynamic_patch |
| 84 | |
| 85 | |
| 86 | def get_internvl_target_ratios( |
| 87 | min_num: int, |
| 88 | max_num: int, |
| 89 | ) -> list[tuple[int, int]]: |
| 90 | target_ratios = { |
| 91 | (i, j) |
| 92 | for n in range(min_num, max_num + 1) |
| 93 | for i in range(1, n + 1) |
| 94 | for j in range(1, n + 1) |
| 95 | if min_num <= i * j <= max_num |
| 96 | } |
| 97 | return sorted(target_ratios, key=lambda x: x[0] * x[1]) |
| 98 | |
| 99 | |
| 100 | def calculate_internvl_targets( |
| 101 | *, |
| 102 | orig_width: int, |
| 103 | orig_height: int, |
| 104 | target_ratios: list[tuple[int, int]], |
| 105 | image_size: int, |
| 106 | use_thumbnail: bool, |
| 107 | ) -> tuple[int, int, int]: |
| 108 | aspect_ratio = orig_width / orig_height |
| 109 | |
| 110 | # find the closest aspect ratio to the target |
| 111 | target_aspect_ratio = find_closest_aspect_ratio( |
| 112 | aspect_ratio, |
| 113 | target_ratios, |
| 114 | width=orig_width, |
| 115 | height=orig_height, |
| 116 | image_size=image_size, |
| 117 | ) |
| 118 | |
| 119 | # calculate the target width and height |
| 120 | target_width = image_size * target_aspect_ratio[0] |
| 121 | target_height = image_size * target_aspect_ratio[1] |
| 122 | blocks = target_aspect_ratio[0] * target_aspect_ratio[1] |
| 123 | |
| 124 | # add thumbnail image if num_blocks != 1 |
| 125 | if use_thumbnail and blocks != 1: |
| 126 | blocks += 1 |
| 127 | |
| 128 | return blocks, target_width, target_height |
| 129 | |
| 130 | |
| 131 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B |
| 132 | def dynamic_preprocess_internvl( |
| 133 | image: Image.Image, |
| 134 | *, |
| 135 | target_ratios: list[tuple[int, int]], |
| 136 | image_size: int, |
| 137 | use_thumbnail: bool, |
| 138 | ) -> list[Image.Image]: |
| 139 | orig_width, orig_height = image.size |
| 140 | |
| 141 | # calculate the number of blocks without thumbnail |
| 142 | blocks, target_width, target_height = calculate_internvl_targets( |
| 143 | orig_width=orig_width, |
| 144 | orig_height=orig_height, |
| 145 | target_ratios=target_ratios, |
| 146 | image_size=image_size, |
| 147 | use_thumbnail=False, |
| 148 | ) |
| 149 | |
| 150 | # resize the image |
| 151 | resized_img = image.resize((target_width, target_height)) |
| 152 | processed_images = [] |
| 153 | for i in range(blocks): |
| 154 | box = ( |
| 155 | (i % (target_width // image_size)) * image_size, |
| 156 | (i // (target_width // image_size)) * image_size, |
| 157 | ((i % (target_width // image_size)) + 1) * image_size, |
| 158 | ((i // (target_width // image_size)) + 1) * image_size, |
| 159 | ) |
| 160 | # split the image |
| 161 | split_img = resized_img.crop(box) |
| 162 | processed_images.append(split_img) |
| 163 | |
| 164 | assert len(processed_images) == blocks |
| 165 | |
| 166 | if use_thumbnail and len(processed_images) != 1: |
| 167 | thumbnail_img = image.resize((image_size, image_size)) |
| 168 | processed_images.append(thumbnail_img) |
| 169 | |
| 170 | return processed_images |
| 171 | |
| 172 | |
| 173 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B |
| 174 | def image_to_pixel_values_internvl( |
| 175 | image: Image.Image, |
| 176 | *, |
| 177 | input_size: int, |
| 178 | min_num: int, |
| 179 | max_num: int, |
| 180 | use_thumbnail: bool, |
| 181 | ) -> torch.Tensor: |
| 182 | target_ratios = get_internvl_target_ratios(min_num, max_num) |
| 183 | |
| 184 | transform = build_transform(input_size=input_size) |
| 185 | images = dynamic_preprocess_internvl( |
| 186 | image, |
| 187 | target_ratios=target_ratios, |
| 188 | image_size=input_size, |
| 189 | use_thumbnail=use_thumbnail, |
| 190 | ) |
| 191 | |
| 192 | pixel_values = torch.stack([transform(image) for image in images]) |
| 193 | return pixel_values |
| 194 | |
| 195 | |
| 196 | # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B |
| 197 | def video_to_pixel_values_internvl( |
| 198 | video: npt.NDArray, |
| 199 | *, |
| 200 | input_size: int, |
| 201 | min_num: int, |
| 202 | max_num: int, |
| 203 | use_thumbnail: bool, |
| 204 | ) -> torch.Tensor: |
| 205 | target_ratios = get_internvl_target_ratios(min_num, max_num) |
| 206 | |
| 207 | transform = build_transform(input_size=input_size) |
| 208 | frames_list = list[Image.Image]() |
| 209 | for frame in video: |
| 210 | pil_frame = dynamic_preprocess_internvl( |
| 211 | Image.fromarray(frame, mode="RGB"), |
| 212 | target_ratios=target_ratios, |
| 213 | image_size=input_size, |
| 214 | use_thumbnail=use_thumbnail, |
| 215 | ) |
| 216 | assert len(pil_frame) == 1 |
| 217 | frames_list.extend(pil_frame) |
| 218 | |
| 219 | pixel_values = torch.stack([transform(image) for image in frames_list]) |
| 220 | return pixel_values |
| 221 | |
| 222 | |
| 223 | class InternVLImageProcessor(ImageProcessingMixin): |
| 224 | def __init__( |
| 225 | self, |
| 226 | image_size: int, |
| 227 | min_dynamic_patch: int, |
| 228 | max_dynamic_patch: int, |
| 229 | dynamic_image_size: bool, |
| 230 | use_thumbnail: bool, |
| 231 | ) -> None: |
| 232 | self.image_size = image_size |
| 233 | self.min_dynamic_patch = min_dynamic_patch |
| 234 | self.max_dynamic_patch = max_dynamic_patch |
| 235 | self.dynamic_image_size = dynamic_image_size |
| 236 | self.use_thumbnail = use_thumbnail |
| 237 | |
| 238 | def resolve_min_max_num( |
| 239 | self, |
| 240 | *, |
| 241 | min_dynamic_patch: int | None = None, |
| 242 | max_dynamic_patch: int | None = None, |
| 243 | dynamic_image_size: bool | None = None, |
| 244 | use_thumbnail: bool | None = None, |
| 245 | ) -> tuple[int, int]: |
| 246 | if min_dynamic_patch is None: |
| 247 | min_dynamic_patch = self.min_dynamic_patch |
| 248 | if max_dynamic_patch is None: |
| 249 | max_dynamic_patch = self.max_dynamic_patch |
| 250 | if dynamic_image_size is None: |
| 251 | dynamic_image_size = self.dynamic_image_size |
| 252 | if use_thumbnail is None: |
| 253 | use_thumbnail = self.use_thumbnail |
| 254 | |
| 255 | return resolve_internvl_min_max_num( |
| 256 | min_dynamic_patch=min_dynamic_patch, |
| 257 | max_dynamic_patch=max_dynamic_patch, |
| 258 | dynamic_image_size=dynamic_image_size, |
| 259 | use_thumbnail=use_thumbnail, |
| 260 | ) |
| 261 | |
| 262 | def _images_to_pixel_values_lst( |
| 263 | self, |
| 264 | images: list[Image.Image], |
| 265 | min_dynamic_patch: int | None = None, |
| 266 | max_dynamic_patch: int | None = None, |
| 267 | dynamic_image_size: bool | None = None, |
| 268 | ) -> list[torch.Tensor]: |
| 269 | if min_dynamic_patch is None: |
| 270 | min_dynamic_patch = self.min_dynamic_patch |
| 271 | if max_dynamic_patch is None: |
| 272 | max_dynamic_patch = self.max_dynamic_patch |
| 273 | if dynamic_image_size is None: |
| 274 | dynamic_image_size = self.dynamic_image_size |
| 275 | |
| 276 | min_num, max_num = resolve_internvl_min_max_num( |
| 277 | min_dynamic_patch=min_dynamic_patch, |
| 278 | max_dynamic_patch=max_dynamic_patch, |
| 279 | dynamic_image_size=dynamic_image_size, |
| 280 | use_thumbnail=False, # Applied in image_to_pixel_values |
| 281 | ) |
| 282 | |
| 283 | return [ |
| 284 | image_to_pixel_values_internvl( |
| 285 | image, |
| 286 | input_size=self.image_size, |
| 287 | min_num=min_num, |
| 288 | max_num=max_num, |
| 289 | use_thumbnail=self.use_thumbnail, |
| 290 | ) |
| 291 | for image in images |
| 292 | ] |
| 293 | |
| 294 | def __call__( |
| 295 | self, |
| 296 | images: Image.Image | list[Image.Image], |
| 297 | *, |
| 298 | min_dynamic_patch: int | None = None, |
| 299 | max_dynamic_patch: int | None = None, |
| 300 | dynamic_image_size: bool | None = None, |
| 301 | return_tensors: str | TensorType | None = None, |
| 302 | **kwargs, |
| 303 | ) -> BatchFeature: |
| 304 | images_lst = [images] if not isinstance(images, list) else images |
| 305 | |
| 306 | pixel_values_lst = self._images_to_pixel_values_lst( |
| 307 | images_lst, |
| 308 | min_dynamic_patch=min_dynamic_patch, |
| 309 | max_dynamic_patch=max_dynamic_patch, |
| 310 | dynamic_image_size=dynamic_image_size, |
| 311 | ) |
| 312 | |
| 313 | image_inputs = { |
| 314 | "pixel_values_flat": torch.cat(pixel_values_lst), |
| 315 | "image_num_patches": torch.tensor([len(item) for item in pixel_values_lst]), |
| 316 | } |
| 317 | return BatchFeature(image_inputs, tensor_type=return_tensors) |
| 318 | |
| 319 | |
| 320 | class InternVLVideoProcessor(BaseVideoProcessor): |
| 321 | def __init__( |
| 322 | self, |
| 323 | image_size: int, |
| 324 | ) -> None: |
| 325 | self.image_size = image_size |
| 326 | |
| 327 | def _videos_to_pixel_values_lst( |
| 328 | self, |
| 329 | videos: list[npt.NDArray], |
| 330 | ) -> list[torch.Tensor]: |
| 331 | return [ |
| 332 | video_to_pixel_values_internvl( |
| 333 | video, |
| 334 | input_size=self.image_size, |
| 335 | min_num=1, |
| 336 | max_num=1, |
| 337 | use_thumbnail=False, |
| 338 | ) |
| 339 | for video in videos |
| 340 | ] |
| 341 | |
| 342 | def __call__( |
| 343 | self, |
| 344 | videos: npt.NDArray | list[npt.NDArray], |
| 345 | *, |
| 346 | return_tensors: str | TensorType | None = None, |
| 347 | **kwargs, |
| 348 | ) -> BatchFeature: |
| 349 | videos_lst = [videos] if not isinstance(videos, list) else videos |
| 350 | |
| 351 | pixel_values_lst = self._videos_to_pixel_values_lst(videos_lst) |
| 352 | |
| 353 | image_inputs = { |
| 354 | "pixel_values_flat_video": torch.cat(pixel_values_lst), |
| 355 | "video_num_patches": torch.tensor([len(item) for item in pixel_values_lst]), |
| 356 | } |
| 357 | return BatchFeature(image_inputs, tensor_type=return_tensors) |
| 358 | |
| 359 | |
| 360 | class InternVLProcessor(ProcessorMixin): |
| 361 | """This model doesn't define its own HF processor, |
| 362 | so we implement our own one here. |
| 363 | |
| 364 | The code to insert image tokens is based on: |
| 365 | https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py#L252 |
| 366 | |
| 367 | Code for video processing is adapted from video example: |
| 368 | https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers |
| 369 | """ |
| 370 | |
| 371 | attributes = ["image_processor", "tokenizer", "video_processor"] |
| 372 | |
| 373 | def __init__( |
| 374 | self, |
| 375 | image_processor: InternVLImageProcessor, |
| 376 | tokenizer: HfTokenizer, |
| 377 | video_processor: InternVLVideoProcessor | None = None, |
| 378 | *, |
| 379 | image_seq_length: int, |
| 380 | start_image_token: str = "<img>", |
| 381 | end_image_token: str = "</img>", |
| 382 | ctx_image_token: str = "<IMG_CONTEXT>", |
| 383 | ctx_video_token: str | None = None, |
| 384 | ) -> None: |
| 385 | self.image_processor = image_processor |
| 386 | self.tokenizer = tokenizer |
| 387 | self.video_processor = video_processor |
| 388 | |
| 389 | self.image_seq_length = image_seq_length |
| 390 | self.start_image_token = start_image_token |
| 391 | self.end_image_token = end_image_token |
| 392 | self.ctx_image_token = ctx_image_token |
| 393 | self.ctx_video_token = ctx_video_token |
| 394 | |
| 395 | self.start_image_token_id = tokenizer.convert_tokens_to_ids(start_image_token) |
| 396 | self.end_image_token_id = tokenizer.convert_tokens_to_ids(end_image_token) |
| 397 | self.ctx_image_token_id = tokenizer.convert_tokens_to_ids(ctx_image_token) |
| 398 | self.ctx_video_token_id = ( |
| 399 | None |
| 400 | if ctx_video_token is None |
| 401 | else tokenizer.convert_tokens_to_ids(ctx_video_token) |
| 402 | ) |
| 403 | |
| 404 | def resolve_target_ratios( |
| 405 | self, |
| 406 | *, |
| 407 | min_dynamic_patch: int | None = None, |
| 408 | max_dynamic_patch: int | None = None, |
| 409 | dynamic_image_size: bool | None = None, |
| 410 | use_thumbnail: bool | None = None, |
| 411 | ) -> list[tuple[int, int]]: |
| 412 | min_num, max_num = self.image_processor.resolve_min_max_num( |
| 413 | min_dynamic_patch=min_dynamic_patch, |
| 414 | max_dynamic_patch=max_dynamic_patch, |
| 415 | dynamic_image_size=dynamic_image_size, |
| 416 | use_thumbnail=use_thumbnail, |
| 417 | ) |
| 418 | |
| 419 | return get_internvl_target_ratios(min_num, max_num) |
| 420 | |
| 421 | def get_num_image_tokens( |
| 422 | self, |
| 423 | *, |
| 424 | image_width: int, |
| 425 | image_height: int, |
| 426 | ) -> int: |
| 427 | image_processor = self.image_processor |
| 428 | target_ratios = self.resolve_target_ratios( |
| 429 | use_thumbnail=False, # Applied in calculate_targets |
| 430 | ) |
| 431 | |
| 432 | num_patches, _, _ = calculate_internvl_targets( |
| 433 | orig_width=image_width, |
| 434 | orig_height=image_height, |
| 435 | image_size=image_processor.image_size, |
| 436 | target_ratios=target_ratios, |
| 437 | use_thumbnail=image_processor.use_thumbnail, |
| 438 | ) |
| 439 | |
| 440 | return num_patches * self.image_seq_length |
| 441 | |
| 442 | def get_image_repl( |
| 443 | self, |
| 444 | num_patches: int | None, |
| 445 | num_features: int | None = None, |
| 446 | ) -> PromptUpdateDetails: |
| 447 | if num_patches is None: |
| 448 | assert num_features is not None |
| 449 | else: |
| 450 | num_features = num_patches * self.image_seq_length |
| 451 | |
| 452 | repl_features = self.ctx_image_token * num_features |
| 453 | repl_full = self.start_image_token + repl_features + self.end_image_token |
| 454 | |
| 455 | full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False) |
| 456 | |
| 457 | return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id) |
| 458 | |
| 459 | def get_video_repl(self, num_patches: int) -> PromptUpdateDetails: |
| 460 | assert self.ctx_video_token is not None |
| 461 | |
| 462 | repl_features = self.ctx_video_token * self.image_seq_length |
| 463 | repl_features_with_sep = ( |
| 464 | self.start_image_token + repl_features + self.end_image_token |
| 465 | ) |
| 466 | # num_patches is equal to num_frames |
| 467 | repl_full = "".join( |
| 468 | [f"Frame{i + 1}: {repl_features_with_sep}" for i in range(num_patches)] |
| 469 | ) |
| 470 | |
| 471 | full_ids = cached_encode(self.tokenizer, repl_full, add_special_tokens=False) |
| 472 | assert self.ctx_video_token_id is not None |
| 473 | |
| 474 | return PromptUpdateDetails.select_token_id(full_ids, self.ctx_video_token_id) |
| 475 | |
| 476 | def __call__( |
| 477 | self, |
| 478 | text: str | list[str] | None = None, |
| 479 | images: Image.Image | list[Image.Image] | None = None, |
| 480 | videos: npt.NDArray | list[npt.NDArray] | None = None, |
| 481 | *, |
| 482 | min_dynamic_patch: int | None = None, |
| 483 | max_dynamic_patch: int | None = None, |
| 484 | dynamic_image_size: bool | None = None, |
| 485 | return_tensors: str | TensorType | None = None, |
| 486 | **kwargs, |
| 487 | ) -> BatchFeature: |
| 488 | if images is not None: |
| 489 | image_inputs = self.image_processor( |
| 490 | images=images, |
| 491 | min_dynamic_patch=min_dynamic_patch, |
| 492 | max_dynamic_patch=max_dynamic_patch, |
| 493 | dynamic_image_size=dynamic_image_size, |
| 494 | return_tensors=return_tensors, |
| 495 | ) |
| 496 | image_num_patches = image_inputs["image_num_patches"] |
| 497 | else: |
| 498 | image_inputs = {} |
| 499 | image_num_patches = [] |
| 500 | |
| 501 | if videos is not None: |
| 502 | if self.video_processor is None: |
| 503 | raise ValueError("This model does not support video inputs") |
| 504 | |
| 505 | video_inputs = self.video_processor( |
| 506 | videos=videos, |
| 507 | return_tensors=return_tensors, |
| 508 | ) |
| 509 | video_num_patches = video_inputs["video_num_patches"] |
| 510 | else: |
| 511 | video_inputs = {} |
| 512 | video_num_patches = [] |
| 513 | |
| 514 | if text is not None: |
| 515 | if not isinstance(text, list): |
| 516 | text = [text] |
| 517 | |
| 518 | if image_inputs: |
| 519 | image_token = "<image>" |
| 520 | image_index = 0 |
| 521 | processed_text = list[str]() |
| 522 | replace_strings = list[str]() |
| 523 | |
| 524 | for prompt in text: |
| 525 | new_prompt = prompt |
| 526 | |
| 527 | while image_token in new_prompt: |
| 528 | new_prompt = new_prompt.replace(image_token, "<placeholder>", 1) |
| 529 | image_repl = self.get_image_repl(image_num_patches[image_index]) |
| 530 | # Convert the token IDs back to text for the |
| 531 | # string-based replacement below |
| 532 | replace_strings.append( |
| 533 | self.tokenizer.decode( |
| 534 | image_repl.full, skip_special_tokens=False |
| 535 | ) |
| 536 | ) |
| 537 | image_index += 1 |
| 538 | |
| 539 | while "<placeholder>" in new_prompt: |
| 540 | replace_str = replace_strings.pop(0) |
| 541 | new_prompt = new_prompt.replace("<placeholder>", replace_str, 1) |
| 542 | |
| 543 | processed_text.append(new_prompt) |
| 544 | |
| 545 | text = processed_text |
| 546 | |
| 547 | if video_inputs: |
| 548 | video_token = "<video>" |
| 549 | video_index = 0 |
| 550 | processed_text = list[str]() |
| 551 | replace_strings = list[str]() |
| 552 | |
| 553 | assert video_token is not None |
| 554 | |
| 555 | for prompt in text: |
| 556 | new_prompt = prompt |
| 557 | |
| 558 | while video_token in new_prompt: |
| 559 | new_prompt = new_prompt.replace(video_token, "<placeholder>", 1) |
| 560 | video_repl = self.get_video_repl(video_num_patches[video_index]) |
| 561 | # Convert the token IDs back to text for the |
| 562 | # string-based replacement below |
| 563 | replace_strings.append( |
| 564 | self.tokenizer.decode( |
| 565 | video_repl.full, skip_special_tokens=False |
| 566 | ) |
| 567 | ) |
| 568 | video_index += 1 |
| 569 | |
| 570 | while "<placeholder>" in new_prompt: |
| 571 | replace_str = replace_strings.pop(0) |
| 572 | new_prompt = new_prompt.replace("<placeholder>", replace_str, 1) |
| 573 | |
| 574 | processed_text.append(new_prompt) |
| 575 | |
| 576 | text = processed_text |
| 577 | |
| 578 | text_inputs = self.tokenizer(text, return_tensors=return_tensors) |
| 579 | else: |
| 580 | text_inputs = {} |
| 581 | |
| 582 | return BatchFeature( |
| 583 | data={**text_inputs, **image_inputs, **video_inputs}, |
| 584 | tensor_type=return_tensors, |
| 585 | ) |