@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de1cb38769
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 17:15 UTC

[Model] Support Skywork-R1V (#15397)

Signed-off-by: jiacai.liu <932997367@qq.com> Co-authored-by: jiacai.liu <932997367@qq.com>

pengyuange committed Mar 29, 2025 at 11:39 UTC de1cb38769e9eb9812fa425c4fdfcf8faa3c420e
12 files changed +1194 -7
docs/source/models/supported_models.md
+7
@@ -921,6 +921,13 @@ See [this page](#generative-models) for more information on how to use generativ
921 * ✅︎
922 * ✅︎
923 * ✅︎
924 +- * `SkyworkR1VChatModel`
925 + * Skywork-R1V-38B
926 + * T + I
927 + * `Skywork/Skywork-R1V-38B`
928 + *
929 + * ✅︎
930 + * ✅︎
931 - * `UltravoxModel`
932 * Ultravox
933 * T + A<sup>E+</sup>
examples/offline_inference/vision_language.py
+36
@@ -804,6 +804,41 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
804 )
805
806
807 +# SkyworkR1V
808 +def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
809 + assert modality == "image"
810 +
811 + model_name = "Skywork/Skywork-R1V-38B"
812 +
813 + engine_args = EngineArgs(
814 + model=model_name,
815 + trust_remote_code=True,
816 + max_model_len=4096,
817 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
818 + )
819 +
820 + tokenizer = AutoTokenizer.from_pretrained(model_name,
821 + trust_remote_code=True)
822 + messages = [[{
823 + 'role': 'user',
824 + 'content': f"<image>\n{question}"
825 + }] for question in questions]
826 + prompts = tokenizer.apply_chat_template(messages,
827 + tokenize=False,
828 + add_generation_prompt=True)
829 +
830 + # Stop tokens for SkyworkR1V
831 + # https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/conversation.py
832 + stop_tokens = ["<｜end▁of▁sentence｜>", "<|endoftext|>"]
833 + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
834 +
835 + return ModelRequestData(
836 + engine_args=engine_args,
837 + prompts=prompts,
838 + stop_token_ids=stop_token_ids,
839 + )
840 +
841 +
842 model_example_map = {
843 "aria": run_aria,
844 "blip-2": run_blip2,
@@ -834,6 +869,7 @@ model_example_map = {
869 "qwen_vl": run_qwen_vl,
870 "qwen2_vl": run_qwen2_vl,
871 "qwen2_5_vl": run_qwen2_5_vl,
872 + "skywork_chat": run_skyworkr1v,
873 }
874
875
tests/models/decoder_only/vision_language/test_models.py
+14
@@ -474,6 +474,20 @@ VLM_TEST_SETTINGS = {
474 vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
475 prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
476 ),
477 + "skywork_r1v": VLMTestInfo(
478 + models=["Skywork/Skywork-R1V-38B"],
479 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
480 + prompt_formatter=lambda img_prompt: f"<｜begin▁of▁sentence｜><｜User｜>\n{img_prompt}<｜Assistant｜><think>\n", # noqa: E501
481 + single_image_prompts=IMAGE_ASSETS.prompts({
482 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
483 + "cherry_blossom": "<image>\nWhat is the season?",
484 + }),
485 + multi_image_prompt="<image>\n<image>\nDescribe the two images in short.", # noqa: E501
486 + max_model_len=4096,
487 + use_tokenizer_eos=True,
488 + patch_hf_runner=model_utils.skyworkr1v_patch_hf_runner,
489 + marks=[large_gpu_mark(min_gb=80)],
490 + ),
491 ### Tensor parallel / multi-gpu broadcast tests
492 "chameleon-broadcast": VLMTestInfo(
493 models=["facebook/chameleon-7b"],
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+57
@@ -376,6 +376,63 @@ def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
376 return hf_model
377
378
379 +def skyworkr1v_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
380 + """Patches and returns an instance of the HfRunner to use for SkyworkR1V."""
381 +
382 + class SkyworkR1VProcessor:
383 + """A simple processor for SkyworkR1V."""
384 +
385 + def __init__(self, hf_runner: HfRunner):
386 + self.num_image_token = hf_runner.model.num_image_token
387 + self.tokenizer = hf_runner.tokenizer
388 +
389 + self.config = AutoConfig.from_pretrained(hf_runner.model_name,
390 + trust_remote_code=True)
391 + self.vision_config = self.config.vision_config
392 + self.use_thumbnail = self.config.use_thumbnail
393 + self.min_num = self.config.min_dynamic_patch
394 + self.max_num = self.config.max_dynamic_patch
395 + self.image_size = self.vision_config.image_size
396 +
397 + def __call__(self, text: str, images: Union[Image, list[Image]],
398 + **kwargs):
399 + from vllm.model_executor.models.skyworkr1v import (
400 + IMG_CONTEXT, IMG_END, IMG_START,
401 + image_to_pixel_values_skyworkr1v)
402 + images = [images] if isinstance(images, Image) else images
403 + pixel_values = [
404 + image_to_pixel_values_skyworkr1v(
405 + image,
406 + input_size=self.image_size,
407 + min_num=self.min_num,
408 + max_num=self.max_num,
409 + use_thumbnail=self.use_thumbnail,
410 + ) for image in images
411 + ]
412 + num_patches_list = [
413 + pixel_value.shape[0] for pixel_value in pixel_values
414 + ]
415 + pixel_values = torch.cat(pixel_values, dim=0)
416 + for num_patches in num_patches_list:
417 + context_tokens = IMG_CONTEXT * self.num_image_token \
418 + * num_patches
419 + image_tokens = IMG_START + context_tokens + IMG_END
420 + text = text.replace('<image>', image_tokens, 1)
421 + prompt = self.tokenizer(text, return_tensors="pt")
422 + prompt.update({"pixel_values": pixel_values})
423 + return prompt
424 +
425 + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
426 + "<IMG_CONTEXT>")
427 + hf_model.model.img_context_token_id = img_context_token_id
428 + hf_model.processor = SkyworkR1VProcessor(hf_model)
429 + hf_model.model.get_output_embeddings = lambda: \
430 + hf_model.model.language_model.get_output_embeddings()
431 + hf_model.model.generate = types.MethodType(_internvl_generate,
432 + hf_model.model)
433 + return hf_model
434 +
435 +
436 def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
437 """Patches and returns an instance of the HfRunner to use for InternVL."""
438
tests/models/multimodal/processing/test_common.py
+5 -4
@@ -262,22 +262,23 @@ def _test_processing_correctness_mistral(
262 "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
263 "meta-llama/Llama-3.2-11B-Vision-Instruct",
264 "TIGER-Lab/Mantis-8B-siglip-llama3",
265 - "mistralai/Pixtral-12B-2409",
266 - "mistral-community/pixtral-12b",
265 "openbmb/MiniCPM-Llama3-V-2_5",
266 "openbmb/MiniCPM-o-2_6",
267 "openbmb/MiniCPM-V-2_6",
268 "allenai/Molmo-7B-D-0924",
269 "allenai/Molmo-7B-O-0924",
270 "nvidia/NVLM-D-72B",
271 + "google/paligemma-3b-mix-224",
272 + "google/paligemma2-3b-ft-docci-448",
273 + "mistralai/Pixtral-12B-2409",
274 + "mistral-community/pixtral-12b",
275 "Qwen/Qwen-VL-Chat",
276 "Qwen/Qwen2-VL-2B-Instruct",
277 "Qwen/Qwen2.5-VL-3B-Instruct",
278 "Qwen/Qwen2-Audio-7B-Instruct",
279 + "Skywork/Skywork-R1V-38B",
280 "fixie-ai/ultravox-v0_5-llama-3_2-1b",
281 "openai/whisper-large-v3",
279 - "google/paligemma-3b-mix-224",
280 - "google/paligemma2-3b-ft-docci-448",
282 ])
283 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
284 @pytest.mark.parametrize("num_batches", [32])
tests/models/registry.py
+1
@@ -294,6 +294,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
294 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
295 "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
296 min_transformers_version="4.49"), # noqa: E501
297 + "SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
298 "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
299 trust_remote_code=True),
300 # [Encoder-decoder]
vllm/entrypoints/chat_utils.py
+1 -1
@@ -496,7 +496,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
496 return self._cached_token_str(self._tokenizer,
497 hf_config.image_token_index)
498 if model_type in ("chameleon", "deepseek_vl_v2", "internvl_chat",
499 - "NVLM_D", "h2ovl_chat"):
499 + "skywork_chat", "NVLM_D", "h2ovl_chat"):
500 return "<image>"
501 if model_type == "mllama":
502 return "<|image|>"
vllm/model_executor/models/registry.py
+1
@@ -190,6 +190,7 @@ _MULTIMODAL_MODELS = {
190 # [Encoder-decoder]
191 "Florence2ForConditionalGeneration": ("florence2", "Florence2ForConditionalGeneration"), # noqa: E501
192 "MllamaForConditionalGeneration": ("mllama", "MllamaForConditionalGeneration"), # noqa: E501
193 + "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
194 "WhisperForConditionalGeneration": ("whisper", "WhisperForConditionalGeneration"), # noqa: E501
195 }
196
vllm/model_executor/models/skyworkr1v.py new
+1014
@@ -0,0 +1,1014 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py
4 +# --------------------------------------------------------
5 +# SkyworkR1V
6 +# Copyright (c) 2025 Skywork
7 +# Licensed under The MIT License [see LICENSE for details]
8 +# --------------------------------------------------------
9 +from abc import ABC, abstractmethod
10 +from collections.abc import Iterable, Mapping, Sequence
11 +from functools import cached_property
12 +from typing import Literal, Optional, Set, Tuple, TypedDict, TypeVar, Union
13 +
14 +import torch
15 +import torch.nn as nn
16 +import torchvision.transforms as T
17 +from PIL import Image
18 +from transformers import BatchEncoding, PretrainedConfig, TensorType
19 +
20 +from vllm.config import VllmConfig
21 +from vllm.model_executor.layers.linear import ReplicatedLinear
22 +from vllm.model_executor.layers.quantization import QuantizationConfig
23 +from vllm.model_executor.layers.quantization.awq import AWQConfig
24 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
25 +from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 + InternVisionPatchModel)
27 +from vllm.model_executor.sampling_metadata import SamplingMetadata
28 +from vllm.multimodal import MULTIMODAL_REGISTRY
29 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
30 + NestedTensors)
31 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 + ImageSize, MultiModalDataItems)
33 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 + BaseProcessingInfo, PromptReplacement,
35 + PromptUpdate, PromptUpdateDetails)
36 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 +from vllm.sequence import IntermediateTensors
38 +from vllm.transformers_utils.tokenizer import AnyTokenizer
39 +
40 +from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
41 +from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
42 + maybe_prefix, merge_multimodal_embeddings)
43 +from .vision import scatter_patch_features, select_patch_features
44 +
45 +IMG_START = '<img>'
46 +IMG_END = '</img>'
47 +IMG_CONTEXT = '<IMG_CONTEXT>'
48 +
49 +IMAGENET_MEAN = (0.485, 0.456, 0.406)
50 +IMAGENET_STD = (0.229, 0.224, 0.225)
51 +
52 +
53 +class SkyworkR1VImagePixelInputs(TypedDict):
54 + type: Literal["pixel_values"]
55 + pixel_values_flat: torch.Tensor
56 + """
57 + Shape:
58 + `(batch_size * num_images * (1 + num_patches), num_channels, height, width)`
59 + """
60 +
61 + num_patches: torch.Tensor
62 + """Shape: `(batch_size * num_images)`"""
63 +
64 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]
65 + """
66 + A boolean mask indicating which image embeddings correspond
67 + to patch tokens.
68 +
69 + Shape: `(batch_size * num_images, num_embeds)`
70 + """
71 +
72 +
73 +class SkyworkR1VImageEmbeddingInputs(TypedDict):
74 + type: Literal["image_embeds"]
75 + data: Union[torch.Tensor, list[torch.Tensor]]
76 + """
77 + A tensor of shape `(num_images, total_image_feature_size, hidden_size)`
78 + or a list of tensors of shape `(total_image_feature_size, hidden_size)`
79 +
80 + `hidden_size` must match the hidden size of language model backbone.
81 + """
82 +
83 +
84 +SkyworkR1VImageInputs = Union[SkyworkR1VImagePixelInputs,
85 + SkyworkR1VImageEmbeddingInputs]
86 +
87 +
88 +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
89 +def build_transform(input_size: int):
90 + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
91 + return T.Compose([
92 + T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
93 + T.Resize((input_size, input_size),
94 + interpolation=T.InterpolationMode.BICUBIC),
95 + T.ToTensor(),
96 + T.Normalize(mean=MEAN, std=STD)
97 + ])
98 +
99 +
100 +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B/
101 +def find_closest_aspect_ratio(
102 + aspect_ratio: float,
103 + target_ratios: list[tuple[int, int]],
104 + *,
105 + width: int,
106 + height: int,
107 + image_size: int,
108 +) -> tuple[int, int]:
109 + best_ratio_diff = float('inf')
110 + best_ratio = (1, 1)
111 + area = width * height
112 + for ratio in target_ratios:
113 + target_aspect_ratio = ratio[0] / ratio[1]
114 + ratio_diff = abs(aspect_ratio - target_aspect_ratio)
115 + if ratio_diff < best_ratio_diff:
116 + best_ratio_diff = ratio_diff
117 + best_ratio = ratio
118 + elif ratio_diff == best_ratio_diff:
119 + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
120 + best_ratio = ratio
121 + return best_ratio
122 +
123 +
124 +def resolve_skyworkr1v_min_max_num(
125 + *,
126 + min_dynamic_patch: int,
127 + max_dynamic_patch: int,
128 + dynamic_image_size: bool,
129 + use_thumbnail: bool,
130 +) -> tuple[int, int]:
131 + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
132 + max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
133 +
134 + if use_thumbnail and max_dynamic_patch != 1:
135 + max_dynamic_patch += 1
136 +
137 + return min_dynamic_patch, max_dynamic_patch
138 +
139 +
140 +def get_skyworkr1v_target_ratios(
141 + min_num: int,
142 + max_num: int,
143 +) -> list[tuple[int, int]]:
144 + target_ratios = {(i, j)
145 + for n in range(min_num, max_num + 1)
146 + for i in range(1, n + 1)
147 + for j in range(1, n + 1) if min_num <= i * j <= max_num}
148 + return sorted(target_ratios, key=lambda x: x[0] * x[1])
149 +
150 +
151 +def calculate_skyworkr1v_targets(
152 + *,
153 + orig_width: int,
154 + orig_height: int,
155 + target_ratios: list[tuple[int, int]],
156 + image_size: int,
157 + use_thumbnail: bool,
158 +) -> tuple[int, int, int]:
159 + aspect_ratio = orig_width / orig_height
160 +
161 + # find the closest aspect ratio to the target
162 + target_aspect_ratio = find_closest_aspect_ratio(
163 + aspect_ratio,
164 + target_ratios,
165 + width=orig_width,
166 + height=orig_height,
167 + image_size=image_size,
168 + )
169 +
170 + # calculate the target width and height
171 + target_width = image_size * target_aspect_ratio[0]
172 + target_height = image_size * target_aspect_ratio[1]
173 + blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
174 +
175 + # add thumbnail image if num_blocks != 1
176 + if use_thumbnail and blocks != 1:
177 + blocks += 1
178 +
179 + return blocks, target_width, target_height
180 +
181 +
182 +def dynamic_preprocess_skyworkr1v(
183 + image: Image.Image,
184 + *,
185 + target_ratios: list[tuple[int, int]],
186 + image_size: int,
187 + use_thumbnail: bool,
188 +) -> list[Image.Image]:
189 + orig_width, orig_height = image.size
190 +
191 + # calculate the number of blocks without thumbnail
192 + blocks, target_width, target_height = calculate_skyworkr1v_targets(
193 + orig_width=orig_width,
194 + orig_height=orig_height,
195 + target_ratios=target_ratios,
196 + image_size=image_size,
197 + use_thumbnail=False,
198 + )
199 +
200 + # resize the image
201 + resized_img = image.resize((target_width, target_height))
202 + processed_images = []
203 + for i in range(blocks):
204 + box = ((i % (target_width // image_size)) * image_size,
205 + (i // (target_width // image_size)) * image_size,
206 + ((i % (target_width // image_size)) + 1) * image_size,
207 + ((i // (target_width // image_size)) + 1) * image_size)
208 + # split the image
209 + split_img = resized_img.crop(box)
210 + processed_images.append(split_img)
211 +
212 + assert len(processed_images) == blocks
213 +
214 + if use_thumbnail and len(processed_images) != 1:
215 + thumbnail_img = image.resize((image_size, image_size))
216 + processed_images.append(thumbnail_img)
217 +
218 + return processed_images
219 +
220 +
221 +# adapted from https://huggingface.co/Skywork/Skywork-R1V-38B
222 +def image_to_pixel_values_skyworkr1v(
223 + image: Image.Image,
224 + *,
225 + input_size: int,
226 + min_num: int,
227 + max_num: int,
228 + use_thumbnail: bool,
229 +) -> torch.Tensor:
230 + target_ratios = get_skyworkr1v_target_ratios(min_num, max_num)
231 +
232 + transform = build_transform(input_size=input_size)
233 + images = dynamic_preprocess_skyworkr1v(
234 + image,
235 + target_ratios=target_ratios,
236 + image_size=input_size,
237 + use_thumbnail=use_thumbnail,
238 + )
239 +
240 + pixel_values = torch.stack([transform(image) for image in images])
241 + return pixel_values
242 +
243 +
244 +class BaseSkyworkR1VProcessor(ABC):
245 + """
246 + This model doesn't define its own HF processor,
247 + so we implement our own one here.
248 +
249 + The code to insert image tokens is based on:
250 + https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/modeling_skywork_chat.py#L252
251 + """
252 +
253 + def __init__(
254 + self,
255 + config: PretrainedConfig,
256 + tokenizer: AnyTokenizer,
257 + *,
258 + min_dynamic_patch: Optional[int] = None,
259 + max_dynamic_patch: Optional[int] = None,
260 + dynamic_image_size: Optional[bool] = None,
261 + ) -> None:
262 + super().__init__()
263 +
264 + self.config = config
265 + self.tokenizer = tokenizer
266 +
267 + image_size: int = config.vision_config.image_size
268 + patch_size: int = config.vision_config.patch_size
269 +
270 + if min_dynamic_patch is None:
271 + min_dynamic_patch = config.min_dynamic_patch
272 + assert isinstance(min_dynamic_patch, int)
273 +
274 + if max_dynamic_patch is None:
275 + max_dynamic_patch = config.max_dynamic_patch
276 + assert isinstance(max_dynamic_patch, int)
277 +
278 + if dynamic_image_size is None:
279 + dynamic_image_size = config.dynamic_image_size
280 + assert isinstance(dynamic_image_size, bool)
281 +
282 + self.num_image_token = int(
283 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
284 + self.image_size = image_size
285 + self.min_dynamic_patch = min_dynamic_patch
286 + self.max_dynamic_patch = max_dynamic_patch
287 + self.dynamic_image_size = dynamic_image_size
288 + self.use_thumbnail: bool = config.use_thumbnail
289 +
290 + @property
291 + @abstractmethod
292 + def image_token_id(self) -> int:
293 + raise NotImplementedError
294 +
295 + @abstractmethod
296 + def get_image_repl(
297 + self,
298 + feature_size: int,
299 + num_patches: Optional[int],
300 + ) -> PromptUpdateDetails[str]:
301 + raise NotImplementedError
302 +
303 + def resolve_min_max_num(
304 + self,
305 + *,
306 + min_dynamic_patch: Optional[int] = None,
307 + max_dynamic_patch: Optional[int] = None,
308 + dynamic_image_size: Optional[bool] = None,
309 + use_thumbnail: Optional[bool] = None,
310 + ) -> tuple[int, int]:
311 + min_dynamic_patch = (self.min_dynamic_patch if min_dynamic_patch
312 + is None else min_dynamic_patch)
313 + max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
314 + is None else max_dynamic_patch)
315 + dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
316 + is None else dynamic_image_size)
317 + use_thumbnail = (self.use_thumbnail
318 + if use_thumbnail is None else use_thumbnail)
319 +
320 + return resolve_skyworkr1v_min_max_num(
321 + min_dynamic_patch=min_dynamic_patch,
322 + max_dynamic_patch=max_dynamic_patch,
323 + dynamic_image_size=dynamic_image_size,
324 + use_thumbnail=use_thumbnail,
325 + )
326 +
327 + def resolve_target_ratios(
328 + self,
329 + *,
330 + min_dynamic_patch: Optional[int] = None,
331 + max_dynamic_patch: Optional[int] = None,
332 + dynamic_image_size: Optional[bool] = None,
333 + use_thumbnail: Optional[bool] = None,
334 + ) -> list[tuple[int, int]]:
335 + min_num, max_num = self.resolve_min_max_num(
336 + min_dynamic_patch=min_dynamic_patch,
337 + max_dynamic_patch=max_dynamic_patch,
338 + dynamic_image_size=dynamic_image_size,
339 + use_thumbnail=use_thumbnail,
340 + )
341 +
342 + return get_skyworkr1v_target_ratios(min_num, max_num)
343 +
344 + def get_num_image_tokens(
345 + self,
346 + *,
347 + image_width: int,
348 + image_height: int,
349 + ) -> int:
350 + target_ratios = self.resolve_target_ratios(
351 + use_thumbnail=False, # Applied in calculate_targets
352 + )
353 +
354 + num_patches, _, _ = calculate_skyworkr1v_targets(
355 + orig_width=image_width,
356 + orig_height=image_height,
357 + image_size=self.image_size,
358 + target_ratios=target_ratios,
359 + use_thumbnail=self.use_thumbnail,
360 + )
361 +
362 + return num_patches * self.num_image_token
363 +
364 + def _images_to_pixel_values_lst(
365 + self,
366 + images: list[Image.Image],
367 + min_dynamic_patch: Optional[int] = None,
368 + max_dynamic_patch: Optional[int] = None,
369 + dynamic_image_size: Optional[bool] = None,
370 + ) -> list[torch.Tensor]:
371 + min_num, max_num = self.resolve_min_max_num(
372 + min_dynamic_patch=min_dynamic_patch,
373 + max_dynamic_patch=max_dynamic_patch,
374 + dynamic_image_size=dynamic_image_size,
375 + use_thumbnail=False, # Applied in image_to_pixel_values
376 + )
377 +
378 + return [
379 + image_to_pixel_values_skyworkr1v(
380 + image,
381 + input_size=self.image_size,
382 + min_num=min_num,
383 + max_num=max_num,
384 + use_thumbnail=self.use_thumbnail,
385 + ) for image in images
386 + ]
387 +
388 + def __call__(
389 + self,
390 + text: Optional[Union[str, list[str]]] = None,
391 + images: Optional[Union[Image.Image, list[Image.Image]]] = None,
392 + min_dynamic_patch: Optional[int] = None,
393 + max_dynamic_patch: Optional[int] = None,
394 + dynamic_image_size: Optional[bool] = None,
395 + return_tensors: Optional[Union[str, TensorType]] = None,
396 + ) -> Mapping[str, NestedTensors]:
397 + if text is None:
398 + text = []
399 + if not isinstance(text, list):
400 + text = [text]
401 + if images is None:
402 + images = []
403 + if not isinstance(images, list):
404 + images = [images]
405 +
406 + if len(images) == 0:
407 + image_inputs = {}
408 + else:
409 + pixel_values_lst = self._images_to_pixel_values_lst(
410 + images,
411 + min_dynamic_patch=min_dynamic_patch,
412 + max_dynamic_patch=max_dynamic_patch,
413 + dynamic_image_size=dynamic_image_size,
414 + )
415 + image_inputs: dict[str, NestedTensors] = {
416 + "pixel_values_flat":
417 + torch.cat(pixel_values_lst),
418 + "image_num_patches":
419 + torch.tensor([len(item) for item in pixel_values_lst]),
420 + }
421 +
422 + tokenizer = self.tokenizer
423 + image_token_id = self.image_token_id
424 +
425 + embed_is_patch = list[torch.Tensor]()
426 +
427 + for pixel_values in pixel_values_lst:
428 + num_patches = pixel_values.shape[0]
429 + feature_size = num_patches * self.num_image_token
430 +
431 + image_repl = self.get_image_repl(feature_size, num_patches)
432 + feature_tokens = tokenizer.encode(image_repl.features,
433 + add_special_tokens=False)
434 +
435 + text = [t.replace('<image>', image_repl.full, 1) for t in text]
436 + embed_is_patch.append(
437 + torch.tensor(feature_tokens) == image_token_id)
438 +
439 + image_inputs["embed_is_patch"] = embed_is_patch
440 +
441 + text_inputs = self.tokenizer(text)
442 +
443 + return {
444 + **BatchEncoding(text_inputs, tensor_type=return_tensors),
445 + **image_inputs,
446 + }
447 +
448 +
449 +class SkyworkR1VProcessor(BaseSkyworkR1VProcessor):
450 +
451 + @property
452 + def image_token_id(self) -> int:
453 + return self.tokenizer.get_vocab()[IMG_CONTEXT]
454 +
455 + def get_image_repl(
456 + self,
457 + feature_size: int,
458 + num_patches: Optional[int],
459 + ) -> PromptUpdateDetails[str]:
460 + repl_features = IMG_CONTEXT * feature_size
461 + repl_full = IMG_START + repl_features + IMG_END
462 +
463 + return PromptUpdateDetails(full=repl_full, features=repl_features)
464 +
465 +
466 +class BaseSkyworkR1VProcessingInfo(BaseProcessingInfo):
467 +
468 + @abstractmethod
469 + def get_hf_processor(
470 + self,
471 + *,
472 + min_dynamic_patch: Optional[int] = None,
473 + max_dynamic_patch: Optional[int] = None,
474 + dynamic_image_size: Optional[bool] = None,
475 + **kwargs: object,
476 + ) -> BaseSkyworkR1VProcessor:
477 + raise NotImplementedError
478 +
479 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
480 + return {"image": None}
481 +
482 + def get_mm_max_tokens_per_item(
483 + self,
484 + seq_len: int,
485 + mm_counts: Mapping[str, int],
486 + ) -> Mapping[str, int]:
487 + return {"image": self.get_max_image_tokens()}
488 +
489 + def get_num_image_tokens(
490 + self,
491 + *,
492 + image_width: int,
493 + image_height: int,
494 + processor: Optional[BaseSkyworkR1VProcessor],
495 + ) -> int:
496 + if processor is None:
497 + processor = self.get_hf_processor()
498 +
499 + return processor.get_num_image_tokens(
500 + image_width=image_width,
501 + image_height=image_height,
502 + )
503 +
504 + def get_max_image_tokens(self) -> int:
505 + target_width, target_height = self.get_image_size_with_most_features()
506 +
507 + return self.get_num_image_tokens(
508 + image_width=target_width,
509 + image_height=target_height,
510 + processor=None,
511 + )
512 +
513 + def get_image_size_with_most_features(self) -> ImageSize:
514 + processor = self.get_hf_processor()
515 +
516 + base_size = processor.image_size
517 + target_ratios = processor.resolve_target_ratios()
518 +
519 + largest_feature_size, largest_feature_pinpoint = 0, None
520 + for wr, hr in target_ratios:
521 + width, height = base_size * wr, base_size * hr
522 +
523 + feat_size = self.get_num_image_tokens(
524 + image_width=width,
525 + image_height=height,
526 + processor=processor,
527 + )
528 + if feat_size > largest_feature_size:
529 + largest_feature_size = feat_size
530 + largest_feature_pinpoint = ImageSize(width=width,
531 + height=height)
532 +
533 + if largest_feature_size == 0 or largest_feature_pinpoint is None:
534 + raise ValueError("Cannot have a largest feature size of 0!")
535 +
536 + return largest_feature_pinpoint
537 +
538 +
539 +_I = TypeVar("_I", bound=BaseSkyworkR1VProcessingInfo)
540 +
541 +
542 +class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
543 +
544 + def get_dummy_processor_inputs(
545 + self,
546 + seq_len: int,
547 + mm_counts: Mapping[str, int],
548 + ) -> ProcessorInputs:
549 + target_width, target_height = \
550 + self.info.get_image_size_with_most_features()
551 + num_images = mm_counts.get("image", 0)
552 +
553 + mm_data = {
554 + "image":
555 + self._get_dummy_images(width=target_width,
556 + height=target_height,
557 + num_images=num_images)
558 + }
559 +
560 + return ProcessorInputs(
561 + prompt_text="<image>" * num_images,
562 + mm_data=mm_data,
563 + )
564 +
565 +
566 +class SkyworkR1VMultiModalProcessor(BaseMultiModalProcessor[_I]):
567 +
568 + def _call_hf_processor(
569 + self,
570 + prompt: str,
571 + mm_data: Mapping[str, object],
572 + mm_kwargs: Mapping[str, object],
573 + ) -> Mapping[str, NestedTensors]:
574 + processed_outputs = super()._call_hf_processor(
575 + prompt=prompt,
576 + mm_data=mm_data,
577 + mm_kwargs=mm_kwargs,
578 + )
579 +
580 + hf_processor = self.info.get_hf_processor(**mm_kwargs)
581 + image_token_id = hf_processor.image_token_id
582 +
583 + # Since there may be extra tokens in the feature placeholders,
584 + # we need to pass the image token ID to the model to select the
585 + # tokens to merge from the vision encoder outputs
586 + processed_outputs["image_token_id"] = torch.tensor(image_token_id)
587 +
588 + return processed_outputs
589 +
590 + def _get_mm_fields_config(
591 + self,
592 + hf_inputs: Mapping[str, NestedTensors],
593 + hf_processor_mm_kwargs: Mapping[str, object],
594 + ) -> Mapping[str, MultiModalFieldConfig]:
595 + image_num_patches = hf_inputs.get("image_num_patches", torch.empty(0))
596 + num_images = len(image_num_patches)
597 +
598 + return dict(
599 + pixel_values_flat=MultiModalFieldConfig.flat_from_sizes(
600 + "image", image_num_patches),
601 + image_num_patches=MultiModalFieldConfig.batched("image"),
602 + embed_is_patch=MultiModalFieldConfig.batched("image"),
603 + image_embeds=MultiModalFieldConfig.batched("image"),
604 + image_token_id=MultiModalFieldConfig.shared("image", num_images),
605 + )
606 +
607 + def _get_prompt_updates(
608 + self,
609 + mm_items: MultiModalDataItems,
610 + hf_processor_mm_kwargs: Mapping[str, object],
611 + out_mm_kwargs: MultiModalKwargs,
612 + ) -> Sequence[PromptUpdate]:
613 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
614 +
615 + if "image_num_patches" in out_mm_kwargs:
616 + image_num_patches = out_mm_kwargs["image_num_patches"]
617 + assert isinstance(image_num_patches, torch.Tensor)
618 + image_num_patches = image_num_patches.tolist()
619 + elif "image_embeds" in out_mm_kwargs:
620 + # TODO: Use image size information in dictionary embedding inputs
621 + # to compute num_patches (similar to Qwen2-VL)
622 + image_num_patches = [None] * len(out_mm_kwargs["image_embeds"])
623 + else:
624 + image_num_patches = []
625 +
626 + def get_replacement_skyworkr1v(item_idx: int):
627 + images = mm_items.get_items(
628 + "image", (ImageEmbeddingItems, ImageProcessorItems))
629 +
630 + if isinstance(images, ImageEmbeddingItems):
631 + feature_size = images.get_feature_size(item_idx)
632 + else:
633 + image_size = images.get_image_size(item_idx)
634 + feature_size = self.info.get_num_image_tokens(
635 + image_width=image_size.width,
636 + image_height=image_size.height,
637 + processor=hf_processor,
638 + )
639 +
640 + num_patches = image_num_patches[item_idx]
641 + if num_patches is not None:
642 + assert isinstance(num_patches, int)
643 +
644 + return hf_processor.get_image_repl(feature_size, num_patches)
645 +
646 + return [
647 + PromptReplacement(
648 + modality="image",
649 + target="<image>",
650 + replacement=get_replacement_skyworkr1v,
651 + )
652 + ]
653 +
654 +
655 +class SkyworkR1VProcessingInfo(BaseSkyworkR1VProcessingInfo):
656 +
657 + def get_hf_processor(
658 + self,
659 + *,
660 + min_dynamic_patch: Optional[int] = None,
661 + max_dynamic_patch: Optional[int] = None,
662 + dynamic_image_size: Optional[bool] = None,
663 + **kwargs: object,
664 + ) -> SkyworkR1VProcessor:
665 + if min_dynamic_patch is not None:
666 + kwargs["min_dynamic_patch"] = min_dynamic_patch
667 + if max_dynamic_patch is not None:
668 + kwargs["max_dynamic_patch"] = max_dynamic_patch
669 + if dynamic_image_size is not None:
670 + kwargs["dynamic_image_size"] = dynamic_image_size
671 +
672 + return self.ctx.init_processor(
673 + SkyworkR1VProcessor,
674 + config=self.get_hf_config(),
675 + tokenizer=self.get_tokenizer(),
676 + **kwargs,
677 + )
678 +
679 +
680 +@MULTIMODAL_REGISTRY.register_processor(
681 + SkyworkR1VMultiModalProcessor,
682 + info=SkyworkR1VProcessingInfo,
683 + dummy_inputs=SkyworkR1VDummyInputsBuilder)
684 +class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
685 +
686 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
687 + super().__init__()
688 +
689 + config = vllm_config.model_config.hf_config
690 + quant_config = vllm_config.quant_config
691 + multimodal_config = vllm_config.model_config.multimodal_config
692 +
693 + self.config = config
694 + self.multimodal_config = multimodal_config
695 + self._patch_quant_config(config, quant_config)
696 +
697 + image_size = config.force_image_size or config.vision_config.image_size
698 + patch_size = config.vision_config.patch_size
699 + self.patch_size = patch_size
700 + self.num_image_token = int(
701 + (image_size // patch_size)**2 * (config.downsample_ratio**2))
702 + self.downsample_ratio = config.downsample_ratio
703 + self.ps_version = config.ps_version
704 +
705 + self.llm_arch_name = config.text_config.architectures[0]
706 + self.is_mono = self.llm_arch_name == 'SkyworkLM2VEForCausalLM'
707 + self.vision_model = self._init_vision_model(
708 + config,
709 + quant_config=quant_config,
710 + is_mono=self.is_mono,
711 + prefix=maybe_prefix(prefix, "vision_model"),
712 + )
713 +
714 + self.language_model = init_vllm_registered_model(
715 + vllm_config=vllm_config,
716 + hf_config=config.text_config,
717 + prefix=maybe_prefix(prefix, "language_model"),
718 + )
719 +
720 + self.mlp1 = self._init_mlp1(config)
721 +
722 + self.img_context_token_id = None
723 + self.visual_token_mask = None
724 + self.make_empty_intermediate_tensors = (
725 + self.language_model.make_empty_intermediate_tensors)
726 +
727 + def _patch_quant_config(self, config: PretrainedConfig,
728 + quant_config: QuantizationConfig):
729 + # the awq models from OpenGVLab missing `modules_to_not_convert`
730 + # patch the quant_config to add `modules_to_not_convert` back
731 + if isinstance(quant_config, AWQConfig):
732 + text_config = config.text_config
733 + llm_quant_config = getattr(text_config, "quantization_config",
734 + None)
735 + if (not quant_config.modules_to_not_convert) and \
736 + (llm_quant_config is not None):
737 + quant_config.modules_to_not_convert.append("vision_model")
738 +
739 + @cached_property
740 + def sampler(self):
741 + if hasattr(self.language_model, "sampler"):
742 + return self.language_model.sampler
743 +
744 + return get_sampler()
745 +
746 + def _init_vision_model(
747 + self,
748 + config: PretrainedConfig,
749 + quant_config: Optional[QuantizationConfig],
750 + *,
751 + is_mono: bool,
752 + prefix: str,
753 + ):
754 + if not is_mono:
755 + vision_feature_layer = config.select_layer
756 + if vision_feature_layer < 0:
757 + num_hidden_layers = config.vision_config.num_hidden_layers \
758 + + vision_feature_layer + 1
759 + else:
760 + num_hidden_layers = vision_feature_layer + 1
761 +
762 + return InternVisionModel(
763 + config.vision_config,
764 + quant_config=quant_config,
765 + num_hidden_layers_override=num_hidden_layers,
766 + prefix=prefix,
767 + )
768 + else:
769 + return InternVisionPatchModel(config.vision_config)
770 +
771 + def _init_mlp1(self, config: PretrainedConfig) -> nn.Sequential:
772 + vit_hidden_size = config.vision_config.hidden_size
773 + llm_hidden_size = config.text_config.hidden_size
774 +
775 + return nn.Sequential(
776 + nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio)**2),
777 + ReplicatedLinear(vit_hidden_size *
778 + int(1 / self.downsample_ratio)**2,
779 + llm_hidden_size,
780 + return_bias=False),
781 + nn.GELU(),
782 + ReplicatedLinear(llm_hidden_size,
783 + llm_hidden_size,
784 + return_bias=False),
785 + )
786 +
787 + def pixel_shuffle(self, x, scale_factor=0.5):
788 + n, w, h, c = x.size()
789 + # N, W, H, C --> N, W, H * scale, C // scale
790 + x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
791 + # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
792 + x = x.permute(0, 2, 1, 3).contiguous()
793 + x = x.view(n, int(h * scale_factor), int(w * scale_factor),
794 + int(c / (scale_factor * scale_factor)))
795 + if self.ps_version == 'v1':
796 + pass
797 + else:
798 + x = x.permute(0, 2, 1, 3).contiguous()
799 + return x
800 +
801 + def extract_feature(self, pixel_values: torch.Tensor) -> torch.Tensor:
802 + vit_embeds = self.vision_model(pixel_values=pixel_values)
803 + vit_embeds = vit_embeds[:, 1:, :]
804 +
805 + h = w = int(vit_embeds.shape[1]**0.5)
806 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
807 + vit_embeds = self.pixel_shuffle(vit_embeds,
808 + scale_factor=self.downsample_ratio)
809 + vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1,
810 + vit_embeds.shape[-1])
811 + vit_embeds = self.mlp1(vit_embeds)
812 + return vit_embeds
813 +
814 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
815 +
816 + h = w = self.config.vision_config.image_size
817 + expected_dims = (3, h, w)
818 +
819 + def _validate_shape(d: torch.Tensor):
820 + actual_dims = tuple(d.shape)
821 +
822 + if actual_dims != expected_dims:
823 + expected_expr = str(expected_dims)
824 + raise ValueError(
825 + "The expected shape of pixel values per image per batch "
826 + f" per patch is {expected_expr}. "
827 + f"You supplied {tuple(d.shape)}.")
828 +
829 + for d in data:
830 + _validate_shape(d)
831 +
832 + return data
833 +
834 + def _parse_and_validate_image_input(
835 + self, **kwargs: object) -> Optional[SkyworkR1VImageInputs]:
836 + pixel_values_flat = kwargs.pop("pixel_values_flat", None)
837 + image_num_patches = kwargs.pop("image_num_patches", None)
838 + embed_is_patch = kwargs.pop("embed_is_patch", None)
839 + image_embeds = kwargs.pop("image_embeds", None)
840 +
841 + if pixel_values_flat is None and image_embeds is None:
842 + return None
843 +
844 + if image_embeds is not None:
845 + if not isinstance(image_embeds, (torch.Tensor, list)):
846 + raise ValueError("Incorrect type of image embeddings. "
847 + f"Got type: {type(image_embeds)}")
848 +
849 + return SkyworkR1VImageEmbeddingInputs(
850 + type="image_embeds",
851 + data=flatten_bn(image_embeds),
852 + )
853 +
854 + image_token_id = kwargs["image_token_id"]
855 + assert isinstance(image_token_id, torch.Tensor)
856 + self.img_context_token_id = image_token_id.flatten().unique().item()
857 +
858 + if pixel_values_flat is not None:
859 + if not isinstance(pixel_values_flat, (torch.Tensor, list)):
860 + raise ValueError("Incorrect type of pixel values. "
861 + f"Got type: {type(pixel_values_flat)}")
862 +
863 + if not isinstance(image_num_patches, (torch.Tensor, list)):
864 + raise ValueError("Incorrect type of image_num_patches. "
865 + f"Got type: {type(image_num_patches)}")
866 +
867 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
868 + raise ValueError("Incorrect type of embed_is_patch. "
869 + f"Got type: {type(embed_is_patch)}")
870 +
871 + pixel_values_flat = flatten_bn(pixel_values_flat, concat=True)
872 + image_num_patches = flatten_bn(image_num_patches, concat=True)
873 + embed_is_patch = flatten_bn(embed_is_patch)
874 +
875 + return SkyworkR1VImagePixelInputs(
876 + type="pixel_values",
877 + pixel_values_flat=self._validate_pixel_values(
878 + pixel_values_flat),
879 + num_patches=image_num_patches,
880 + embed_is_patch=embed_is_patch,
881 + )
882 +
883 + raise AssertionError("This line should be unreachable.")
884 +
885 + def _process_image_input(
886 + self,
887 + image_input: SkyworkR1VImageInputs,
888 + ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]:
889 + if image_input["type"] == "image_embeds":
890 + return image_input["data"]
891 +
892 + assert self.vision_model is not None
893 +
894 + image_embeds = self.extract_feature(image_input["pixel_values_flat"])
895 +
896 + num_patches = image_input["num_patches"]
897 +
898 + # Only one image in the current batch
899 + if len(num_patches) == 1:
900 + return image_embeds.view(
901 + -1, self.config.text_config.hidden_size).unsqueeze(0)
902 +
903 + # NOTE: Image embeddings are split into separate tensors for each image
904 + # by the size of each embedding.
905 + feature_size = image_embeds.shape[1]
906 + image_embeds = image_embeds.view(-1,
907 + self.config.text_config.hidden_size)
908 + image_feature_sizes = [
909 + num_patches * feature_size for num_patches in num_patches
910 + ]
911 + return image_embeds.split(image_feature_sizes)
912 +
913 + def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:
914 + if self.is_mono:
915 + self.visual_token_mask = (
916 + input_ids == self.img_context_token_id).reshape(-1, 1)
917 + else:
918 + self.visual_token_mask = None
919 +
920 + def get_multimodal_embeddings(
921 + self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
922 + image_input = self._parse_and_validate_image_input(**kwargs)
923 + if image_input is None:
924 + return None
925 +
926 + image_features = self._process_image_input(image_input)
927 +
928 + if image_input["type"] != "pixel_values":
929 + return image_features
930 +
931 + return scatter_patch_features(
932 + image_features,
933 + image_input["embed_is_patch"],
934 + )
935 +
936 + def get_input_embeddings(
937 + self,
938 + input_ids: torch.Tensor,
939 + multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
940 + ) -> torch.Tensor:
941 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
942 + if multimodal_embeddings is not None:
943 + assert self.img_context_token_id is not None
944 + self._set_visual_token_mask(input_ids)
945 + inputs_embeds = merge_multimodal_embeddings(
946 + input_ids,
947 + inputs_embeds,
948 + select_patch_features(multimodal_embeddings),
949 + self.img_context_token_id,
950 + )
951 + return inputs_embeds
952 +
953 + def forward(
954 + self,
955 + input_ids: torch.Tensor,
956 + positions: torch.Tensor,
957 + intermediate_tensors: Optional[IntermediateTensors] = None,
958 + inputs_embeds: Optional[torch.Tensor] = None,
959 + **kwargs: object,
960 + ) -> Union[SamplerOutput, IntermediateTensors]:
961 +
962 + if intermediate_tensors is not None:
963 + input_ids = None
964 + inputs_embeds = None
965 +
966 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
967 + # condition is for v0 compatibility.
968 + elif inputs_embeds is None:
969 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
970 + inputs_embeds = self.get_input_embeddings(input_ids,
971 + vision_embeddings)
972 + input_ids = None
973 +
974 + forward_kwargs = {
975 + "input_ids": input_ids,
976 + "positions": positions,
977 + "intermediate_tensors": intermediate_tensors,
978 + "inputs_embeds": inputs_embeds,
979 + }
980 +
981 + # Only required if the model is mono-architecture
982 + if self.visual_token_mask is not None:
983 + forward_kwargs.update(
984 + {"visual_token_mask": self.visual_token_mask})
985 + self.visual_token_mask = None
986 +
987 + hidden_states = self.language_model.model(**forward_kwargs)
988 + return hidden_states
989 +
990 + def compute_logits(
991 + self,
992 + hidden_states: torch.Tensor,
993 + sampling_metadata: SamplingMetadata,
994 + ) -> Optional[torch.Tensor]:
995 + return self.language_model.compute_logits(hidden_states,
996 + sampling_metadata)
997 +
998 + def sample(
999 + self,
1000 + logits: torch.Tensor,
1001 + sampling_metadata: SamplingMetadata,
1002 + ) -> Optional[SamplerOutput]:
1003 + return self.language_model.sample(logits, sampling_metadata)
1004 +
1005 + def load_weights(self, weights: Iterable[Tuple[str,
1006 + torch.Tensor]]) -> Set[str]:
1007 + skip_prefixes = [
1008 + "action_embed", "temporal_embed", "track_embed",
1009 + "track_embed_decoder", "box_token", "cg_criterion", "cg_model",
1010 + "loc_encoder", "loc_decoder", "sam", "temporal_token",
1011 + "track_token"
1012 + ]
1013 + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
1014 + return loader.load_weights(weights)
vllm/transformers_utils/config.py
+3 -2
@@ -37,8 +37,8 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
37 MLPSpeculatorConfig, MPTConfig,
38 NemotronConfig, NVLM_D_Config,
39 Olmo2Config, RWConfig,
40 - SolarConfig, Telechat2Config,
41 - UltravoxConfig)
40 + SkyworkR1VChatConfig, SolarConfig,
41 + Telechat2Config, UltravoxConfig)
42 # yapf: enable
43 from vllm.transformers_utils.utils import check_gguf_file
44 from vllm.utils import resolve_obj_by_qualname
@@ -76,6 +76,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
76 "NVLM_D": NVLM_D_Config,
77 "olmo2": Olmo2Config,
78 "solar": SolarConfig,
79 + "skywork_chat": SkyworkR1VChatConfig,
80 "telechat": Telechat2Config,
81 "ultravox": UltravoxConfig,
82 **_CONFIG_REGISTRY_OVERRIDE_HF
vllm/transformers_utils/configs/__init__.py
+2
@@ -20,6 +20,7 @@ from vllm.transformers_utils.configs.mpt import MPTConfig
20 from vllm.transformers_utils.configs.nemotron import NemotronConfig
21 from vllm.transformers_utils.configs.nvlm_d import NVLM_D_Config
22 from vllm.transformers_utils.configs.olmo2 import Olmo2Config
23 +from vllm.transformers_utils.configs.skyworkr1v import SkyworkR1VChatConfig
24 from vllm.transformers_utils.configs.solar import SolarConfig
25 from vllm.transformers_utils.configs.telechat2 import Telechat2Config
26 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
@@ -42,6 +43,7 @@ __all__ = [
43 "NemotronConfig",
44 "NVLM_D_Config",
45 "Olmo2Config",
46 + "SkyworkR1VChatConfig",
47 "SolarConfig",
48 "Telechat2Config",
49 "UltravoxConfig",
vllm/transformers_utils/configs/skyworkr1v.py new
+53
@@ -0,0 +1,53 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# Adapted from
4 +# https://huggingface.co/Skywork/Skywork-R1V-38B/blob/main/configuration_skywork_chat.py
5 +# --------------------------------------------------------
6 +# SkyworkR1V
7 +# Copyright (c) 2025 Skywork
8 +# Licensed under The MIT License [see LICENSE for details]
9 +# --------------------------------------------------------
10 +from transformers.configuration_utils import PretrainedConfig
11 +
12 +
13 +class SkyworkR1VChatConfig(PretrainedConfig):
14 + model_type = 'internvl_chat'
15 + is_composition = True
16 +
17 + def __init__(self,
18 + vision_config=None,
19 + llm_config=None,
20 + use_backbone_lora=0,
21 + use_llm_lora=0,
22 + select_layer=-1,
23 + force_image_size=None,
24 + downsample_ratio=0.5,
25 + template=None,
26 + dynamic_image_size=False,
27 + use_thumbnail=False,
28 + ps_version='v1',
29 + min_dynamic_patch=1,
30 + max_dynamic_patch=6,
31 + **kwargs):
32 + super().__init__(**kwargs)
33 +
34 + if vision_config is None:
35 + vision_config = {}
36 +
37 + if llm_config is None:
38 + llm_config = {}
39 +
40 + self.vision_config = PretrainedConfig(**vision_config)
41 + self.text_config = PretrainedConfig(**llm_config)
42 +
43 + self.use_backbone_lora = use_backbone_lora
44 + self.use_llm_lora = use_llm_lora
45 + self.select_layer = select_layer
46 + self.force_image_size = force_image_size
47 + self.downsample_ratio = downsample_ratio
48 + self.template = template
49 + self.dynamic_image_size = dynamic_image_size
50 + self.use_thumbnail = use_thumbnail
51 + self.ps_version = ps_version # pixel shuffle version
52 + self.min_dynamic_patch = min_dynamic_patch
53 + self.max_dynamic_patch = max_dynamic_patch