| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # adapted from https://huggingface.co/nvidia/NVLM-D-72B/blob/main/modeling_nvlm_d.py |
| 5 | # -------------------------------------------------------- |
| 6 | # NVLM-D |
| 7 | # Copyright (c) 2024 NVIDIA |
| 8 | # Licensed under Apache 2.0 License [see LICENSE for details] |
| 9 | # -------------------------------------------------------- |
| 10 | from vllm.multimodal.processing import PromptUpdateDetails, cached_encode |
| 11 | from vllm.tokenizers.hf import HfTokenizer |
| 12 | |
| 13 | from .internvl import InternVLImageProcessor, InternVLProcessor |
| 14 | |
| 15 | |
| 16 | class NVLMProcessor(InternVLProcessor): |
| 17 | def __init__( |
| 18 | self, |
| 19 | image_processor: InternVLImageProcessor, |
| 20 | tokenizer: HfTokenizer, |
| 21 | *, |
| 22 | image_seq_length: int, |
| 23 | start_image_token: str = "<Image>", |
| 24 | end_image_token: str = "</Image>", |
| 25 | ctx_image_token: str = "<|vision_pad|>", |
| 26 | ) -> None: |
| 27 | super().__init__( |
| 28 | image_processor=image_processor, |
| 29 | tokenizer=tokenizer, |
| 30 | image_seq_length=image_seq_length, |
| 31 | start_image_token=start_image_token, |
| 32 | end_image_token=end_image_token, |
| 33 | ctx_image_token=ctx_image_token, |
| 34 | ) |
| 35 | |
| 36 | def get_image_repl( |
| 37 | self, |
| 38 | num_patches: int | None, |
| 39 | num_features: int | None = None, |
| 40 | ) -> PromptUpdateDetails: |
| 41 | if num_patches is None: |
| 42 | raise NotImplementedError("Embedding inputs are not supported") |
| 43 | |
| 44 | num_features = num_patches * self.image_seq_length |
| 45 | |
| 46 | tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)] |
| 47 | if self.image_processor.use_thumbnail: |
| 48 | tile_pos_identifiers += ["<tile_global_thumbnail>"] |
| 49 | |
| 50 | context_size = num_features // num_patches |
| 51 | features = "".join( |
| 52 | (identifier + self.ctx_image_token * context_size) |
| 53 | for identifier in tile_pos_identifiers |
| 54 | ) |
| 55 | |
| 56 | # We include the start and end as well because "<Image><tile" is |
| 57 | # tokenized as ["<Image", "><", "tile"], resulting in assertion error |
| 58 | # when trying to find "<tile" as a subsequence of "<Image><tile" |
| 59 | repl = self.start_image_token + features + self.end_image_token |
| 60 | |
| 61 | full_ids = cached_encode(self.tokenizer, repl, add_special_tokens=False) |
| 62 | |
| 63 | return PromptUpdateDetails.select_token_id(full_ids, self.ctx_image_token_id) |