| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | from transformers import SmolVLMProcessor |
| 6 | |
| 7 | from vllm.config import VllmConfig |
| 8 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 9 | |
| 10 | from .idefics3 import Idefics3DummyInputsBuilder as SmolVLMDummyInputsBuilder |
| 11 | from .idefics3 import Idefics3ForConditionalGeneration, Idefics3ProcessingInfo |
| 12 | from .idefics3 import Idefics3MultiModalProcessor as SmolVLMMultiModalProcessor |
| 13 | |
| 14 | |
| 15 | class SmolVLMProcessingInfo(Idefics3ProcessingInfo): |
| 16 | def get_hf_processor(self, **kwargs: object) -> SmolVLMProcessor: |
| 17 | return self.ctx.get_hf_processor(SmolVLMProcessor, **kwargs) |
| 18 | |
| 19 | def _get_image_token(self, processor: SmolVLMProcessor) -> tuple[str, str, str]: |
| 20 | image_token = processor.image_token |
| 21 | fake_image_token = processor.fake_image_token |
| 22 | global_image_token = processor.global_image_token |
| 23 | return image_token, fake_image_token, global_image_token |
| 24 | |
| 25 | |
| 26 | @MULTIMODAL_REGISTRY.register_processor( |
| 27 | SmolVLMMultiModalProcessor, |
| 28 | info=SmolVLMProcessingInfo, |
| 29 | dummy_inputs=SmolVLMDummyInputsBuilder, |
| 30 | ) |
| 31 | class SmolVLMForConditionalGeneration(Idefics3ForConditionalGeneration): |
| 32 | def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): |
| 33 | super().__init__( |
| 34 | vllm_config=vllm_config, |
| 35 | prefix=prefix, |
| 36 | ) |