| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # QianfanOCR is built on InternVL with a Qwen3 language backbone. |
| 5 | # The model architecture and weights are fully compatible with InternVLChatModel, |
| 6 | # only the config model_type / architectures strings differ. |
| 7 | |
| 8 | from transformers import PreTrainedConfig |
| 9 | |
| 10 | from vllm.model_executor.layers.quantization import QuantizationConfig |
| 11 | from vllm.model_executor.layers.quantization.fp8 import Fp8Config |
| 12 | from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig |
| 13 | from vllm.model_executor.layers.quantization.utils.quant_utils import ( |
| 14 | kFp8Static128BlockSym, |
| 15 | kFp8StaticChannelSym, |
| 16 | kFp8StaticTensorSym, |
| 17 | ) |
| 18 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 19 | from vllm.transformers_utils.processors.internvl import ( |
| 20 | InternVLImageProcessor, |
| 21 | InternVLProcessor, |
| 22 | ) |
| 23 | |
| 24 | from .internvl import ( |
| 25 | BaseInternVLDummyInputsBuilder, |
| 26 | BaseInternVLMultiModalProcessor, |
| 27 | BaseInternVLProcessingInfo, |
| 28 | InternVLChatModel, |
| 29 | ) |
| 30 | |
| 31 | |
| 32 | class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo): |
| 33 | """Image-only ProcessingInfo for QianfanOCR (no video support).""" |
| 34 | |
| 35 | def get_hf_processor(self, **kwargs: object) -> InternVLProcessor: |
| 36 | config = self.get_hf_config() |
| 37 | vision_config = config.vision_config |
| 38 | |
| 39 | merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs) |
| 40 | merged_kwargs.setdefault("image_size", vision_config.image_size) |
| 41 | merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) |
| 42 | merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) |
| 43 | merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) |
| 44 | merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail) |
| 45 | |
| 46 | image_processor = InternVLImageProcessor(**merged_kwargs) |
| 47 | image_size = image_processor.image_size |
| 48 | patch_size = vision_config.patch_size |
| 49 | downsample_ratio = config.downsample_ratio |
| 50 | image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2)) |
| 51 | |
| 52 | return InternVLProcessor( |
| 53 | tokenizer=self.get_tokenizer(), |
| 54 | image_processor=image_processor, |
| 55 | video_processor=None, |
| 56 | image_seq_length=image_seq_length, |
| 57 | ctx_video_token=None, |
| 58 | ) |
| 59 | |
| 60 | |
| 61 | @MULTIMODAL_REGISTRY.register_processor( |
| 62 | BaseInternVLMultiModalProcessor, |
| 63 | info=QianfanOCRProcessingInfo, |
| 64 | dummy_inputs=BaseInternVLDummyInputsBuilder, |
| 65 | ) |
| 66 | class QianfanOCRForConditionalGeneration(InternVLChatModel): |
| 67 | """QianfanOCR multimodal model. |
| 68 | |
| 69 | Identical in structure to InternVLChatModel (InternViT vision encoder + |
| 70 | pixel-shuffle MLP connector + Qwen3 language model). This class exists |
| 71 | solely to register the ``QianfanOCRForConditionalGeneration`` architecture |
| 72 | name that appears in the model's config.json. |
| 73 | """ |
| 74 | |
| 75 | def _patch_quant_config( |
| 76 | self, config: PreTrainedConfig, quant_config: QuantizationConfig | None |
| 77 | ) -> None: |
| 78 | super()._patch_quant_config(config, quant_config) |
| 79 | # ignore vit layers to preserve model performance |
| 80 | if isinstance(quant_config, Fp8Config) or ( |
| 81 | isinstance(quant_config, OnlineQuantizationConfig) |
| 82 | and quant_config.args.linear is not None |
| 83 | and quant_config.args.linear.weight |
| 84 | in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym) |
| 85 | ): |
| 86 | _FP8_IGNORED_LAYERS = [ |
| 87 | *( |
| 88 | layer |
| 89 | for i in range(config.vision_config.num_hidden_layers) |
| 90 | for layer in [ |
| 91 | f"vision_model.encoder.layers.{i}.attn.qkv", |
| 92 | f"vision_model.encoder.layers.{i}.attn.proj", |
| 93 | f"vision_model.encoder.layers.{i}.mlp.fc1", |
| 94 | f"vision_model.encoder.layers.{i}.mlp.fc2", |
| 95 | ] |
| 96 | ), |
| 97 | "language_model.lm_head", |
| 98 | "mlp1.1", |
| 99 | "mlp1.3", |
| 100 | ] |
| 101 | quant_config.ignored_layers = list(quant_config.ignored_layers) |
| 102 | for layer in _FP8_IGNORED_LAYERS: |
| 103 | if layer not in quant_config.ignored_layers: |
| 104 | quant_config.ignored_layers.append(layer) |