| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # Copyright 2025 Bytedance Ltd. and/or its affiliates. |
| 4 | """BAGEL processor for image and text inputs.""" |
| 5 | |
| 6 | from transformers.feature_extraction_utils import BatchFeature |
| 7 | from transformers.image_utils import ImageInput |
| 8 | from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack |
| 9 | from transformers.tokenization_utils_base import PreTokenizedInput, TextInput |
| 10 | |
| 11 | |
| 12 | class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] |
| 13 | _defaults = { |
| 14 | "images_kwargs": { |
| 15 | "return_tensors": "pt", |
| 16 | }, |
| 17 | } |
| 18 | |
| 19 | |
| 20 | class BagelProcessor(ProcessorMixin): |
| 21 | """Constructs a BAGEL processor which wraps a |
| 22 | SigLIP image processor and a Qwen2 tokenizer. |
| 23 | """ |
| 24 | |
| 25 | attributes = ["image_processor", "tokenizer"] |
| 26 | image_processor_class = "SiglipImageProcessor" |
| 27 | tokenizer_class = "AutoTokenizer" |
| 28 | |
| 29 | def __call__( |
| 30 | self, |
| 31 | text: TextInput |
| 32 | | PreTokenizedInput |
| 33 | | list[TextInput] |
| 34 | | list[PreTokenizedInput] = None, |
| 35 | images: ImageInput = None, |
| 36 | **kwargs: Unpack[BagelProcessorKwargs], |
| 37 | ): |
| 38 | """Prepare one or more sequences and images for the model.""" |
| 39 | output_kwargs = self._merge_kwargs( |
| 40 | BagelProcessorKwargs, |
| 41 | tokenizer_init_kwargs=self.tokenizer.init_kwargs, |
| 42 | **kwargs, |
| 43 | ) |
| 44 | |
| 45 | if images is not None: |
| 46 | # Process images with the image processor |
| 47 | pixel_values = self.image_processor( |
| 48 | images, **output_kwargs["images_kwargs"] |
| 49 | ) |
| 50 | else: |
| 51 | pixel_values = {} |
| 52 | |
| 53 | text_inputs = ( |
| 54 | self.tokenizer(text, **output_kwargs["text_kwargs"]) |
| 55 | if text is not None |
| 56 | else {} |
| 57 | ) |
| 58 | |
| 59 | return BatchFeature(data={**pixel_values, **text_inputs}) |
| 60 | |
| 61 | def batch_decode(self, *args, **kwargs): |
| 62 | """Forward all arguments to `Qwen2TokenizerFast.batch_decode`.""" |
| 63 | return self.tokenizer.batch_decode(*args, **kwargs) |
| 64 | |
| 65 | def decode(self, *args, **kwargs): |
| 66 | """This method forwards all its arguments to Qwen2TokenizerFast's decode.""" |
| 67 | return self.tokenizer.decode(*args, **kwargs) |
| 68 | |
| 69 | @property |
| 70 | def model_input_names(self): |
| 71 | tokenizer_input_names = self.tokenizer.model_input_names |
| 72 | image_processor_input_names = self.image_processor.model_input_names |
| 73 | return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) |