A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model][VLM] Support R-4B Model (#23246)
Signed-off-by: yannqi <yannqi@qq.com> Signed-off-by: 杨奇(yann qi) <51905299+yannqi@users.noreply.github.com> Signed-off-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: yannqiyang <yannqiyang@tencent.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
杨奇(yann qi) committed
Aug 21, 2025 at 12:08 UTC
655a09f6538e6b09af23771dcc4fcebd72a15b23
7 files changed
+165
docs/models/supported_models.md
+1
@@ -652,6 +652,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
652
| `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
653
| `Qwen2_5_VLForConditionalGeneration` | Qwen2.5-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
654
| `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen2.5-Omni-7B` | | ✅︎ | ✅︎ |
655
+| `RForConditionalGeneration` | R-VL-4B | T + I<sup>E+</sup> | `YannQi/R-4B` | | ✅︎ | ✅︎ |
656
| `SkyworkR1VChatModel` | Skywork-R1V-38B | T + I | `Skywork/Skywork-R1V-38B` | | ✅︎ | ✅︎ |
657
| `SmolVLMForConditionalGeneration` | SmolVLM2 | T + I | `SmolVLM2-2.2B-Instruct` | ✅︎ | | ✅︎ |
658
| `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ | ✅︎ |
examples/offline_inference/vision_language.py
+23
@@ -1436,6 +1436,28 @@ def run_qwen2_5_omni(questions: list[str], modality: str):
1436
)
1437
1438
1439
+# R-4B
1440
+def run_r_vl(questions: list[str], modality: str) -> ModelRequestData:
1441
+ assert modality == "image"
1442
+ model_name = "YannQi/R-4B"
1443
+
1444
+ prompts = [
1445
+ f"<|im_start|>user <image>\n{question}<|im_end|><|im_start|>assistant\n"
1446
+ for question in questions
1447
+ ]
1448
+
1449
+ engine_args = EngineArgs(
1450
+ model=model_name,
1451
+ max_model_len=16384,
1452
+ limit_mm_per_prompt={modality: 1},
1453
+ )
1454
+
1455
+ return ModelRequestData(
1456
+ engine_args=engine_args,
1457
+ prompts=prompts,
1458
+ )
1459
+
1460
+
1461
# SkyworkR1V
1462
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
1463
assert modality == "image"
@@ -1622,6 +1644,7 @@ model_example_map = {
1644
"qwen2_vl": run_qwen2_vl,
1645
"qwen2_5_vl": run_qwen2_5_vl,
1646
"qwen2_5_omni": run_qwen2_5_omni,
1647
+ "rvl": run_r_vl,
1648
"skywork_chat": run_skyworkr1v,
1649
"smolvlm": run_smolvlm,
1650
"step3": run_step3,
examples/offline_inference/vision_language_multi_image.py
+34
@@ -992,6 +992,39 @@ def load_qwen2_5_vl(question: str, image_urls: list[str]) -> ModelRequestData:
992
)
993
994
995
+def load_r_vl(question: str, image_urls: list[str]) -> ModelRequestData:
996
+ model_name = "YannQi/R-4B"
997
+ engine_args = EngineArgs(
998
+ model=model_name,
999
+ max_model_len=16384,
1000
+ max_num_seqs=16,
1001
+ limit_mm_per_prompt={"image": len(image_urls)},
1002
+ )
1003
+
1004
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
1005
+ messages = [
1006
+ {
1007
+ "role": "user",
1008
+ "content": [
1009
+ *placeholders,
1010
+ {"type": "text", "text": question},
1011
+ ],
1012
+ }
1013
+ ]
1014
+
1015
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
1016
+
1017
+ prompt = processor.apply_chat_template(
1018
+ messages, tokenize=False, add_generation_prompt=True
1019
+ )
1020
+
1021
+ return ModelRequestData(
1022
+ engine_args=engine_args,
1023
+ prompt=prompt,
1024
+ image_data=[fetch_image(url) for url in image_urls],
1025
+ )
1026
+
1027
+
1028
def load_smolvlm(question: str, image_urls: list[str]) -> ModelRequestData:
1029
model_name = "HuggingFaceTB/SmolVLM2-2.2B-Instruct"
1030
@@ -1193,6 +1226,7 @@ model_example_map = {
1226
"qwen_vl_chat": load_qwen_vl_chat,
1227
"qwen2_vl": load_qwen2_vl,
1228
"qwen2_5_vl": load_qwen2_5_vl,
1229
+ "rvl": load_r_vl,
1230
"smolvlm": load_smolvlm,
1231
"step3": load_step3,
1232
"tarsier": load_tarsier,
tests/models/multimodal/processing/test_common.py
+1
@@ -316,6 +316,7 @@ def _test_processing_correctness_one(
316
"Qwen/Qwen2.5-VL-3B-Instruct",
317
"Qwen/Qwen2-Audio-7B-Instruct",
318
"Qwen/Qwen2.5-Omni-3B",
319
+ "YannQi/R-4B",
320
"Skywork/Skywork-R1V-38B",
321
"HuggingFaceTB/SmolVLM2-2.2B-Instruct",
322
"stepfun-ai/step3",
tests/models/registry.py
+2
@@ -489,6 +489,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
489
max_model_len=4096),
490
"Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-3B"),
491
"Qwen2_5OmniForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B-AWQ"), # noqa: E501
492
+ "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B",
493
+ trust_remote_code=True),
494
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B",
495
trust_remote_code=True),
496
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct", # noqa: E501
vllm/model_executor/models/registry.py
+1
@@ -217,6 +217,7 @@ _MULTIMODAL_MODELS = {
217
"Idefics3ForConditionalGeneration":("idefics3","Idefics3ForConditionalGeneration"),
218
"SmolVLMForConditionalGeneration": ("smolvlm","SmolVLMForConditionalGeneration"), # noqa: E501
219
"KeyeForConditionalGeneration": ("keye", "KeyeForConditionalGeneration"),
220
+ "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
221
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501
222
"Llama_Nemotron_Nano_VL": ("nemotron_vl", "LlamaNemotronVLChatModel"),
223
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
vllm/model_executor/models/rvl.py
new
+103
@@ -0,0 +1,103 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from collections.abc import Mapping
5
+
6
+import torch
7
+import torch.nn as nn
8
+from transformers.activations import GELUActivation
9
+
10
+from vllm.config import VllmConfig
11
+from vllm.multimodal import MULTIMODAL_REGISTRY
12
+from vllm.multimodal.inputs import MultiModalDataDict
13
+
14
+from .llava_next import (LlavaDummyInputsBuilder, LlavaNextMultiModalProcessor,
15
+ LlavaNextProcessingInfo)
16
+from .llava_onevision import LlavaOnevisionForConditionalGeneration
17
+from .utils import WeightsMapper
18
+
19
+
20
+class RVLProcessingInfo(LlavaNextProcessingInfo):
21
+
22
+ def get_hf_config(self):
23
+ return self.ctx.get_hf_config()
24
+
25
+ def get_hf_processor(self, **kwargs: object):
26
+ return self.ctx.get_hf_processor(**kwargs)
27
+
28
+
29
+class RVLDummyInputsBuilder(LlavaDummyInputsBuilder[RVLProcessingInfo]):
30
+
31
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
32
+ num_images = mm_counts.get("image", 0)
33
+ image_token = "<image>"
34
+
35
+ return image_token * num_images
36
+
37
+ def get_dummy_mm_data(
38
+ self,
39
+ seq_len: int,
40
+ mm_counts: Mapping[str, int],
41
+ ) -> MultiModalDataDict:
42
+ num_images = mm_counts.get("image", 0)
43
+
44
+ target_width, target_height = (
45
+ self.info.get_image_size_with_most_features())
46
+
47
+ return {
48
+ "image":
49
+ self._get_dummy_images(width=target_width,
50
+ height=target_height,
51
+ num_images=num_images),
52
+ }
53
+
54
+
55
+class RVLMultiModalProjector(nn.Module):
56
+
57
+ def __init__(self, config):
58
+ super().__init__()
59
+ self.pre_norm = nn.LayerNorm(config.vision_config.hidden_size,
60
+ eps=1e-06)
61
+ self.linear_1 = nn.Linear(
62
+ config.vision_config.hidden_size,
63
+ config.text_config.hidden_size,
64
+ bias=True,
65
+ )
66
+ self.act = GELUActivation()
67
+ self.linear_2 = nn.Linear(
68
+ config.text_config.hidden_size,
69
+ config.text_config.hidden_size,
70
+ bias=True,
71
+ )
72
+
73
+ def forward(self, image_feature: torch.Tensor) -> torch.Tensor:
74
+ image_feature = self.pre_norm(image_feature)
75
+ hidden_states = self.linear_1(image_feature)
76
+ hidden_states = self.act(hidden_states)
77
+ hidden_states = self.linear_2(hidden_states)
78
+
79
+ return hidden_states
80
+
81
+
82
+@MULTIMODAL_REGISTRY.register_processor(
83
+ LlavaNextMultiModalProcessor,
84
+ info=RVLProcessingInfo,
85
+ dummy_inputs=RVLDummyInputsBuilder,
86
+)
87
+class RForConditionalGeneration(LlavaOnevisionForConditionalGeneration):
88
+
89
+ hf_to_vllm_mapper = WeightsMapper(
90
+ orig_to_new_prefix={
91
+ # mapping for new names in checkpoint saved after transformers
92
+ # v4.52
93
+ "model.language_model.": "language_model.model.",
94
+ "model.vision_tower.": "vision_tower.",
95
+ "model.multi_modal_projector.": "multi_modal_projector.",
96
+ "model.image_newline": "image_newline",
97
+ "lm_head.": "language_model.lm_head.",
98
+ })
99
+
100
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
101
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
102
+ config = vllm_config.model_config.hf_config
103
+ self.multi_modal_projector = RVLMultiModalProjector(config)