A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
[Model] support Qianfan-OCR model (#40136)
Signed-off-by: bairongz <baiyuu.cs@gmail.com> Signed-off-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: zhuangbairong <zhuangbairong@baidu.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
bairongz committed
May 5, 2026 at 18:51 UTC
0a201b60cf70283dddf1fd09e0922613095e60f9
9 files changed
+312
docs/models/supported_models.md
+1
@@ -614,6 +614,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
614
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I<sup>+</sup> / T + A<sup>+</sup> / I<sup>+</sup> + A<sup>+</sup> | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
615
| `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I<sup>+</sup> | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ |
616
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I<sup>+</sup> | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ |
617
+| `QianfanOCRForConditionalGeneration` | QianfanOCR | T + I<sup>E+</sup> | `baidu/Qianfan-OCR`, etc. | ✅︎ | ✅︎ |
618
| `QwenVLForConditionalGeneration`<sup>^</sup> | Qwen-VL | T + I<sup>E+</sup> | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
619
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A<sup>+</sup> | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
620
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
tests/models/multimodal/generation/test_common.py
+10
@@ -928,6 +928,16 @@ VLM_TEST_SETTINGS = {
928
),
929
],
930
),
931
+ "qianfan_ocr": VLMTestInfo(
932
+ models=["baidu/Qianfan-OCR"],
933
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
934
+ prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
935
+ img_idx_to_prompt=lambda idx: "<image>",
936
+ max_model_len=4096,
937
+ use_tokenizer_eos=True,
938
+ auto_cls=AutoModelForImageTextToText,
939
+ hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"),
940
+ ),
941
"qwen_vl": VLMTestInfo(
942
models=["Qwen/Qwen-VL"],
943
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
tests/models/multimodal/generation/vlm_utils/model_utils.py
+91
@@ -1554,3 +1554,94 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
1554
1555
hf_model.model.generate = types.MethodType(_generate, hf_model.model)
1556
return hf_model
1557
+
1558
+
1559
+def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict:
1560
+ """Return hf_model_kwargs with a patched config for QianfanOCR."""
1561
+ from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
1562
+
1563
+ config = QianfanOCRConfig.from_pretrained(model_name)
1564
+ vc = config.vision_config
1565
+ if isinstance(vc.image_size, int):
1566
+ vc.image_size = (vc.image_size, vc.image_size)
1567
+ if isinstance(vc.patch_size, int):
1568
+ vc.patch_size = (vc.patch_size, vc.patch_size)
1569
+ return {"config": config}
1570
+
1571
+
1572
+def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
1573
+ """Patches an HfRunner instance to run QianfanOCR model inference.
1574
+
1575
+ QianfanOCR shares the same architecture as InternVLChatModel, so the
1576
+ patching logic mirrors ``internvl_patch_hf_runner``. The only difference
1577
+ is that we load the config via vllm's registered ``QianfanOCRConfig``
1578
+ instead of relying on ``trust_remote_code``.
1579
+ """
1580
+
1581
+ class QianfanOCRProcessor:
1582
+ def __init__(self, hf_runner: HfRunner):
1583
+ self.tokenizer = hf_runner.tokenizer
1584
+
1585
+ from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig
1586
+
1587
+ self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name)
1588
+ self.vision_config = self.config.vision_config
1589
+ self.use_thumbnail = self.config.use_thumbnail
1590
+ self.min_num = self.config.min_dynamic_patch
1591
+ self.max_num = self.config.max_dynamic_patch
1592
+ self.image_size = self.vision_config.image_size
1593
+
1594
+ # Compute num_image_token from config instead of model attribute,
1595
+ # since the transformers-native model doesn't expose it.
1596
+ image_size = self.config.force_image_size or self.vision_config.image_size
1597
+ patch_size = self.vision_config.patch_size
1598
+ downsample_ratio = self.config.downsample_ratio
1599
+ self.num_image_token = int(
1600
+ (image_size // patch_size) ** 2 * (downsample_ratio**2)
1601
+ )
1602
+
1603
+ def __call__(
1604
+ self,
1605
+ text: str,
1606
+ images: PIL.Image.Image | list[PIL.Image.Image] = None,
1607
+ **kwargs,
1608
+ ):
1609
+ from vllm.transformers_utils.processors.internvl import (
1610
+ image_to_pixel_values_internvl,
1611
+ )
1612
+
1613
+ IMG_START = "<img>"
1614
+ IMG_END = "</img>"
1615
+ IMG_CONTEXT = "<IMG_CONTEXT>"
1616
+
1617
+ images = [images] if isinstance(images, PIL.Image.Image) else images
1618
+ pixel_values_list = [
1619
+ image_to_pixel_values_internvl(
1620
+ image,
1621
+ input_size=self.image_size,
1622
+ min_num=self.min_num,
1623
+ max_num=self.max_num,
1624
+ use_thumbnail=self.use_thumbnail,
1625
+ )
1626
+ for image in images
1627
+ ]
1628
+ num_patches_list = [pv.shape[0] for pv in pixel_values_list]
1629
+ pixel_values = torch.cat(pixel_values_list, dim=0)
1630
+
1631
+ for num_patches in num_patches_list:
1632
+ context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
1633
+ image_tokens = IMG_START + context_tokens + IMG_END
1634
+ text = text.replace("<image>", image_tokens, 1)
1635
+
1636
+ prompt = self.tokenizer(text, return_tensors="pt")
1637
+ prompt.update({"pixel_values": pixel_values})
1638
+ return prompt
1639
+
1640
+ img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
1641
+ hf_model.model.img_context_token_id = img_context_token_id
1642
+ hf_model.processor = QianfanOCRProcessor(hf_model)
1643
+ hf_model.model.get_output_embeddings = (
1644
+ lambda: hf_model.model.language_model.get_output_embeddings()
1645
+ )
1646
+ hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model)
1647
+ return hf_model
tests/models/registry.py
+4
@@ -1264,6 +1264,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1264
},
1265
tokenizer_mode="mistral",
1266
),
1267
+ "QianfanOCRForConditionalGeneration": _HfExamplesInfo(
1268
+ "baidu/Qianfan-OCR",
1269
+ min_transformers_version="5.6.0",
1270
+ ),
1271
"QwenVLForConditionalGeneration": _HfExamplesInfo(
1272
"Qwen/Qwen-VL",
1273
extras={"chat": "Qwen/Qwen-VL-Chat"},
vllm/model_executor/models/qianfan_ocr.py
new
+92
@@ -0,0 +1,92 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# QianfanOCR is built on InternVL with a Qwen3 language backbone.
5
+# The model architecture and weights are fully compatible with InternVLChatModel,
6
+# only the config model_type / architectures strings differ.
7
+
8
+from transformers import PretrainedConfig
9
+
10
+from vllm.model_executor.layers.quantization import QuantizationConfig
11
+from vllm.model_executor.layers.quantization.fp8 import Fp8Config
12
+from vllm.multimodal import MULTIMODAL_REGISTRY
13
+from vllm.transformers_utils.processors.internvl import (
14
+ InternVLImageProcessor,
15
+ InternVLProcessor,
16
+)
17
+
18
+from .internvl import (
19
+ BaseInternVLDummyInputsBuilder,
20
+ BaseInternVLMultiModalProcessor,
21
+ BaseInternVLProcessingInfo,
22
+ InternVLChatModel,
23
+)
24
+
25
+
26
+class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
27
+ """Image-only ProcessingInfo for QianfanOCR (no video support)."""
28
+
29
+ def get_hf_processor(self, **kwargs: object) -> InternVLProcessor:
30
+ config = self.get_hf_config()
31
+ vision_config = config.vision_config
32
+
33
+ kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
34
+ kwargs.setdefault("image_size", vision_config.image_size)
35
+ kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
36
+ kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
37
+ kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
38
+ kwargs.setdefault("use_thumbnail", config.use_thumbnail)
39
+
40
+ image_processor = InternVLImageProcessor(**kwargs)
41
+ image_size = image_processor.image_size
42
+ patch_size = vision_config.patch_size
43
+ downsample_ratio = config.downsample_ratio
44
+ image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2))
45
+
46
+ return InternVLProcessor(
47
+ tokenizer=self.get_tokenizer(),
48
+ image_processor=image_processor,
49
+ video_processor=None,
50
+ image_seq_length=image_seq_length,
51
+ ctx_video_token=None,
52
+ )
53
+
54
+
55
+@MULTIMODAL_REGISTRY.register_processor(
56
+ BaseInternVLMultiModalProcessor,
57
+ info=QianfanOCRProcessingInfo,
58
+ dummy_inputs=BaseInternVLDummyInputsBuilder,
59
+)
60
+class QianfanOCRForConditionalGeneration(InternVLChatModel):
61
+ """QianfanOCR multimodal model.
62
+
63
+ Identical in structure to InternVLChatModel (InternViT vision encoder +
64
+ pixel-shuffle MLP connector + Qwen3 language model). This class exists
65
+ solely to register the ``QianfanOCRForConditionalGeneration`` architecture
66
+ name that appears in the model's config.json.
67
+ """
68
+
69
+ def _patch_quant_config(
70
+ self, config: PretrainedConfig, quant_config: QuantizationConfig
71
+ ) -> None:
72
+ super()._patch_quant_config(config, quant_config)
73
+ # ignore vit layers to preserve model performance
74
+ if isinstance(quant_config, Fp8Config):
75
+ _FP8_IGNORED_LAYERS = [
76
+ *(
77
+ layer
78
+ for i in range(config.vision_config.num_hidden_layers)
79
+ for layer in [
80
+ f"vision_model.encoder.layers.{i}.attn.qkv",
81
+ f"vision_model.encoder.layers.{i}.attn.proj",
82
+ f"vision_model.encoder.layers.{i}.mlp.fc1",
83
+ f"vision_model.encoder.layers.{i}.mlp.fc2",
84
+ ]
85
+ ),
86
+ "language_model.lm_head",
87
+ "mlp1.1",
88
+ "mlp1.3",
89
+ ]
90
+ for layer in _FP8_IGNORED_LAYERS:
91
+ if layer not in quant_config.ignored_layers:
92
+ quant_config.ignored_layers.append(layer)
vllm/model_executor/models/registry.py
+4
@@ -511,6 +511,10 @@ _MULTIMODAL_MODELS = {
511
"Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"),
512
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
513
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"),
514
+ "QianfanOCRForConditionalGeneration": (
515
+ "qianfan_ocr",
516
+ "QianfanOCRForConditionalGeneration",
517
+ ),
518
"QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"),
519
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
520
"Qwen2_5_VLForConditionalGeneration": (
vllm/transformers_utils/config.py
+1
@@ -125,6 +125,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
125
step3_vl="Step3VLConfig",
126
step3_text="Step3TextConfig",
127
step3p5="Step3p5Config",
128
+ qianfan_ocr="QianfanOCRConfig",
129
qwen3_asr="Qwen3ASRConfig",
130
qwen3_next="Qwen3NextConfig",
131
qwen3_5="Qwen3_5Config",
vllm/transformers_utils/configs/__init__.py
+4
@@ -70,6 +70,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
70
"Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl",
71
"Step3TextConfig": "vllm.transformers_utils.configs.step3_vl",
72
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
73
+ "QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr",
74
+ "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr",
75
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
76
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
77
"Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
@@ -135,6 +137,8 @@ __all__ = [
137
"Step3VisionEncoderConfig",
138
"Step3TextConfig",
139
"Step3p5Config",
140
+ "QianfanOCRConfig",
141
+ "QianfanOCRVisionConfig",
142
"Qwen3ASRConfig",
143
"Qwen3NextConfig",
144
"Qwen3_5Config",
vllm/transformers_utils/configs/qianfan_ocr.py
new
+105
@@ -0,0 +1,105 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import Any
5
+
6
+from transformers import PretrainedConfig
7
+from transformers.models.auto import CONFIG_MAPPING
8
+
9
+
10
+class QianfanOCRVisionConfig(PretrainedConfig):
11
+ model_type = "qianfan_ocr_vision"
12
+
13
+ def __init__(
14
+ self,
15
+ hidden_size: int = 1024,
16
+ intermediate_size: int = 4096,
17
+ num_hidden_layers: int = 24,
18
+ num_attention_heads: int = 16,
19
+ num_channels: int = 3,
20
+ image_size: int = 448,
21
+ patch_size: int = 14,
22
+ hidden_act: str = "gelu",
23
+ layer_norm_eps: float = 1e-6,
24
+ attention_dropout: float = 0.0,
25
+ drop_path_rate: float = 0.1,
26
+ qkv_bias: bool = True,
27
+ qk_normalization: bool = False,
28
+ norm_type: str = "layer_norm",
29
+ initializer_range: float = 0.02,
30
+ initializer_factor: float = 0.1,
31
+ use_mask_token: bool = False,
32
+ use_mean_pooling: bool = True,
33
+ **kwargs: Any,
34
+ ):
35
+ super().__init__(**kwargs)
36
+ self.hidden_size = hidden_size
37
+ self.intermediate_size = intermediate_size
38
+ self.num_hidden_layers = num_hidden_layers
39
+ self.num_attention_heads = num_attention_heads
40
+ self.num_channels = num_channels
41
+ self.image_size = image_size
42
+ self.patch_size = patch_size
43
+ self.hidden_act = hidden_act
44
+ self.layer_norm_eps = layer_norm_eps
45
+ self.attention_dropout = attention_dropout
46
+ self.drop_path_rate = drop_path_rate
47
+ self.qkv_bias = qkv_bias
48
+ self.qk_normalization = qk_normalization
49
+ self.norm_type = norm_type
50
+ self.initializer_range = initializer_range
51
+ self.initializer_factor = initializer_factor
52
+ self.use_mask_token = use_mask_token
53
+ self.use_mean_pooling = use_mean_pooling
54
+
55
+
56
+class QianfanOCRConfig(PretrainedConfig):
57
+ model_type = "qianfan_ocr"
58
+
59
+ def __init__(
60
+ self,
61
+ vision_config: dict | None = None,
62
+ text_config: dict | None = None,
63
+ downsample_ratio: float = 0.5,
64
+ dynamic_image_size: bool = True,
65
+ force_image_size: int = 448,
66
+ image_token_id: int = 151671,
67
+ max_dynamic_patch: int = 12,
68
+ min_dynamic_patch: int = 1,
69
+ pad2square: bool = False,
70
+ ps_version: str = "v2",
71
+ select_layer: int = -1,
72
+ template: str = "internvl2_5",
73
+ use_thumbnail: bool = True,
74
+ tie_word_embeddings: bool = False,
75
+ **kwargs: Any,
76
+ ):
77
+ super().__init__(**kwargs)
78
+
79
+ if isinstance(vision_config, dict):
80
+ self.vision_config = QianfanOCRVisionConfig(**vision_config)
81
+ elif vision_config is None:
82
+ self.vision_config = QianfanOCRVisionConfig()
83
+ else:
84
+ self.vision_config = vision_config
85
+
86
+ if isinstance(text_config, dict):
87
+ model_type = text_config.get("model_type", "qwen3")
88
+ self.text_config = CONFIG_MAPPING[model_type](**text_config)
89
+ elif text_config is None:
90
+ self.text_config = CONFIG_MAPPING["qwen3"]()
91
+ else:
92
+ self.text_config = text_config
93
+
94
+ self.downsample_ratio = downsample_ratio
95
+ self.dynamic_image_size = dynamic_image_size
96
+ self.force_image_size = force_image_size
97
+ self.image_token_id = image_token_id
98
+ self.max_dynamic_patch = max_dynamic_patch
99
+ self.min_dynamic_patch = min_dynamic_patch
100
+ self.pad2square = pad2square
101
+ self.ps_version = ps_version
102
+ self.select_layer = select_layer
103
+ self.template = template
104
+ self.use_thumbnail = use_thumbnail
105
+ self.tie_word_embeddings = tie_word_embeddings