@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0bdfd5eb84
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 04:45 UTC

[Bugfix] Vendor MiniCPMV/MiniCPMO processors to unblock Transformers v5 (#44282)

Signed-off-by: guanwei-wu <b08901019@ntu.edu.tw> Signed-off-by: wjinxu <1299461899@qq.com> Co-authored-by: guanwei-wu <b08901019@ntu.edu.tw> Co-authored-by: Cursor <cursoragent@cursor.com>

王金旭 committed Jun 2, 2026 at 22:14 UTC 0bdfd5eb843a8e7fcebd37c155b1db3f1c8a5ee4
7 files changed +965 -16
tests/lora/test_minicpmv_tp.py
-11
@@ -1,10 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from importlib.metadata import version
5 -
4 import pytest
7 -from packaging.version import Version
5
6 import vllm
7 from vllm.assets.image import ImageAsset
@@ -13,14 +10,6 @@ from vllm.platforms import current_platform
10
11 from ..utils import multi_gpu_test
12
16 -pytestmark = pytest.mark.skipif(
17 - Version("5.0") <= Version(version("transformers")),
18 - reason=(
19 - "MiniCPMV custom processor uses tokenizer.im_start_id which is not "
20 - "available on TokenizersBackend in transformers v5.0+"
21 - ),
22 -)
23 -
13 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
14
15 PROMPT_TEMPLATE = (
tests/models/multimodal/generation/test_common.py
-4
@@ -785,8 +785,6 @@ VLM_TEST_SETTINGS = {
785 get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
786 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
787 patch_hf_runner=model_utils.minicpmv_25_patch_hf_runner,
788 - # FIXME: https://huggingface.co/openbmb/MiniCPM-V-2_6/discussions/55
789 - marks=[pytest.mark.skip("HF import fails")],
788 ),
789 "minicpmo_26": VLMTestInfo(
790 models=["openbmb/MiniCPM-o-2_6"],
@@ -800,8 +798,6 @@ VLM_TEST_SETTINGS = {
798 ),
799 hf_output_post_proc=model_utils.minicpmv_trunc_hf_output,
800 patch_hf_runner=model_utils.minicpmo_26_patch_hf_runner,
803 - # FIXME: https://huggingface.co/openbmb/MiniCPM-o-2_6/discussions/49
804 - marks=[pytest.mark.skip("HF import fails")],
801 ),
802 "minicpmv_26": VLMTestInfo(
803 models=["openbmb/MiniCPM-V-2_6"],
vllm/model_executor/models/minicpmo.py
+36 -1
@@ -26,7 +26,7 @@
26
27 import os
28 from collections.abc import Callable, Iterable, Mapping, Sequence
29 -from typing import Annotated, Any, Literal, TypeAlias
29 +from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeAlias
30
31 import torch
32 from torch import nn
@@ -75,6 +75,9 @@ from .utils import AutoWeightsLoader, cast_overflow_tensors, maybe_prefix
75
76 CPU_DEVICE = torch.device("cpu")
77
78 +if TYPE_CHECKING:
79 + from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
80 +
81 if os.getenv("USE_FLAGOS") == "1":
82 import flag_gems
83
@@ -270,6 +273,38 @@ class MiniCPMOMultiModalDataParser(MiniCPMVMultiModalDataParser):
273 class MiniCPMOProcessingInfo(MiniCPMVProcessingInfo):
274 audio_pattern = "(<audio>./</audio>)"
275
276 + def get_hf_processor(self, **kwargs: object) -> "MiniCPMOProcessor":
277 + """Get vendored MiniCPMOProcessor for multimodal (image+audio) inputs.
278 +
279 + Creates a vendored processor that reuses the HF image processor,
280 + feature extractor, and tokenizer; applies the correct audio pooling
281 + configuration; and converts numpy arrays in the image processor to
282 + lists for serialization compatibility. The returned processor is
283 + compatible with Transformers v5.
284 + """
285 + import numpy as np
286 +
287 + hf_processor = self.ctx.get_hf_processor(**kwargs)
288 +
289 + from vllm.transformers_utils.processors.minicpmo import MiniCPMOProcessor
290 +
291 + # Create vendored processor with correct configuration
292 + vendored_processor = MiniCPMOProcessor(
293 + image_processor=hf_processor.image_processor,
294 + feature_extractor=hf_processor.feature_extractor,
295 + tokenizer=hf_processor.tokenizer,
296 + pool_step=self.get_default_audio_pool_step(),
297 + )
298 +
299 + # Convert numpy arrays in image processor to lists for serialization
300 + image_processor = vendored_processor.image_processor
301 + for attr in ("mean", "std"):
302 + val = getattr(image_processor, attr, None)
303 + if val is not None and isinstance(val, np.ndarray):
304 + setattr(image_processor, attr, val.tolist())
305 +
306 + return vendored_processor
307 +
308 def get_data_parser(self):
309 return MiniCPMOMultiModalDataParser(
310 target_sr=self.get_default_audio_sampling_rate(),
vllm/model_executor/models/minicpmv.py
+8
@@ -545,6 +545,14 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
545 def get_hf_processor(self, **kwargs: object):
546 hf_processor = self.ctx.get_hf_processor(**kwargs)
547
548 + from vllm.transformers_utils.processors.minicpmv import MiniCPMVProcessor
549 +
550 + vendored_processor = MiniCPMVProcessor(
551 + image_processor=hf_processor.image_processor,
552 + tokenizer=hf_processor.tokenizer,
553 + )
554 + hf_processor = vendored_processor
555 +
556 # NumPy arrays are considered as Iterable but not Sequence in
557 # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
558 image_processor = hf_processor.image_processor # type: ignore
vllm/transformers_utils/processors/__init__.py
+4
@@ -29,6 +29,8 @@ __all__ = [
29 "KimiAudioProcessor",
30 "KimiK25Processor",
31 "MiMoOmniProcessor",
32 + "MiniCPMOProcessor",
33 + "MiniCPMVProcessor",
34 "MistralCommonPixtralProcessor",
35 "MistralCommonVoxtralProcessor",
36 "NanoNemotronVLProcessor",
@@ -61,6 +63,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
63 "KimiAudioProcessor": "vllm.transformers_utils.processors.kimi_audio",
64 "KimiK25Processor": "vllm.transformers_utils.processors.kimi_k25",
65 "MiMoOmniProcessor": "vllm.transformers_utils.processors.mimo_v2_omni",
66 + "MiniCPMOProcessor": "vllm.transformers_utils.processors.minicpmo",
67 + "MiniCPMVProcessor": "vllm.transformers_utils.processors.minicpmv",
68 "MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
69 "MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
70 "Moondream3Processor": "vllm.transformers_utils.processors.moondream3",
vllm/transformers_utils/processors/minicpmo.py new
+603
@@ -0,0 +1,603 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# coding=utf-8
4 +# Copyright 2025 The OpenBMB Team. All rights reserved.
5 +#
6 +# Licensed under the Apache License, Version 2.0 (the "License");
7 +# you may not use this file except in compliance with the License.
8 +# You may obtain a copy of the License at
9 +#
10 +# http://www.apache.org/licenses/LICENSE-2.0
11 +#
12 +# Unless required by applicable law or agreed to in writing, software
13 +# distributed under the License is distributed on an "AS IS" BASIS,
14 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 +# See the License for the specific language governing permissions and
16 +# limitations under the License.
17 +"""
18 +Processor class for MiniCPMO.
19 +"""
20 +
21 +import math
22 +from typing import Literal, TypeAlias
23 +
24 +import numpy as np
25 +import regex
26 +import torch
27 +import torchaudio
28 +from transformers.image_processing_utils import BatchFeature
29 +from transformers.image_utils import ImageInput
30 +from transformers.processing_utils import ProcessorMixin
31 +from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
32 +from transformers.utils import TensorType
33 +
34 +MiniCPMOBatchFeature: TypeAlias = BatchFeature
35 +
36 +
37 +class MiniCPMOProcessor(ProcessorMixin):
38 + r"""
39 + Constructs a MiniCPMV processor which wraps a MiniCPMV image
40 + processor and a MiniCPMV tokenizer into a single processor.
41 +
42 + [`MiniCPMVProcessor`] offers all the functionalities of
43 + [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
44 + [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
45 + for more information.
46 +
47 + Args:
48 + image_processor ([`MiniCPMVImageProcessor`], *optional*):
49 + The image processor is a required input.
50 + tokenizer ([`LlamaTokenizerWrapper`], *optional*):
51 + The tokenizer is a required input.
52 + """
53 +
54 + attributes = ["image_processor", "feature_extractor", "tokenizer"]
55 + feature_extractor_class = "WhisperFeatureExtractor"
56 + image_processor_class = "AutoImageProcessor"
57 + tokenizer_class = "AutoTokenizer"
58 +
59 + def __init__(
60 + self,
61 + image_processor=None,
62 + feature_extractor=None,
63 + tokenizer=None,
64 + pool_step=2,
65 + ):
66 + super().__init__(image_processor, feature_extractor, tokenizer)
67 + self.version = image_processor.version
68 + self.pool_step = pool_step
69 +
70 + def _safe_get_token_id(self, attr_name, default_token_str):
71 + """Get token ID safely, with fallback to default."""
72 + val = getattr(self.tokenizer, attr_name, None)
73 + if val is None:
74 + val = self.tokenizer.convert_tokens_to_ids(default_token_str)
75 + if val is None:
76 + return -1
77 + return val
78 +
79 + def _safe_get_token_str(self, attr_name, default_token_str):
80 + """Get token string safely, with fallback to default."""
81 + return getattr(self.tokenizer, attr_name, default_token_str)
82 +
83 + def __call__(
84 + self,
85 + text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
86 + images: ImageInput = None,
87 + audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]] = None,
88 + audio_parts: list | None = None,
89 + max_length: int | None = None,
90 + do_pad: bool | None = True,
91 + max_slice_nums: int | None = None,
92 + use_image_id: bool = True,
93 + chunk_input: bool = False,
94 + return_tensors: str | TensorType | None = TensorType.PYTORCH,
95 + sampling_rate: int | None = 16000,
96 + **kwargs,
97 + ) -> MiniCPMOBatchFeature:
98 + if images is not None:
99 + image_inputs = self.image_processor(
100 + images,
101 + do_pad=do_pad,
102 + max_slice_nums=max_slice_nums,
103 + return_tensors=return_tensors,
104 + )
105 + else:
106 + image_inputs = None
107 +
108 + if audios is not None:
109 + audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract(
110 + audios, audio_parts, chunk_input, sampling_rate
111 + )
112 + else:
113 + audio_features, audio_feature_lens, audio_phs = [], [], []
114 +
115 + model_inputs = self._convert_omni_to_inputs(
116 + image_inputs,
117 + audio_phs,
118 + text,
119 + max_slice_nums=max_slice_nums,
120 + use_image_id=use_image_id,
121 + max_length=max_length,
122 + **kwargs,
123 + )
124 +
125 + model_inputs["audio_features"] = audio_features
126 + model_inputs["audio_feature_lens"] = audio_feature_lens
127 +
128 + return MiniCPMOBatchFeature(data={**model_inputs})
129 +
130 + def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length):
131 + pool_step = self.pool_step
132 + feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length)
133 +
134 + feature_lens = (feature_lens - 1) // 2 + 1
135 + output_lens = (feature_lens - pool_step) // pool_step + 1
136 +
137 + audio_start = getattr(self.tokenizer, "audio_start", "<audio>")
138 + audio_end = getattr(self.tokenizer, "audio_end", "</audio>")
139 +
140 + if chunk_input:
141 + fbank_feat_in_chunk = int(chunk_length * 100)
142 + cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1
143 + audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1
144 + num_audio_chunks = (
145 + output_lens + audio_embeds_in_chunk - 1
146 + ) // audio_embeds_in_chunk
147 +
148 + place_holders = ""
149 + total_unk_len = 0
150 + for _ in range(num_audio_chunks):
151 + unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len)
152 + place_holders += audio_start + "<unk>" * unk_len + audio_end
153 + total_unk_len += unk_len
154 + audio_placeholder = place_holders
155 + else:
156 + audio_placeholder = audio_start + "<unk>" * output_lens + audio_end
157 +
158 + return audio_placeholder
159 +
160 + def audio_feature_extract(
161 + self,
162 + audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]],
163 + audio_parts: list | None = None,
164 + chunk_input: bool | None = False,
165 + sampling_rate: int | None = None,
166 + chunk_length: int | None = 1,
167 + **kwargs,
168 + ):
169 + if isinstance(audios, np.ndarray):
170 + audios_list = [[audios]]
171 + elif isinstance(audios[0], np.ndarray):
172 + audios_list = [audios]
173 + else:
174 + audios_list = audios
175 +
176 + if audio_parts is not None:
177 + assert len(audio_parts) == len(audios_list)
178 + for parts, audios in zip(audio_parts, audios_list):
179 + assert len(parts) == len(audios)
180 +
181 + audio_feature_lens_list = []
182 + audio_ph_list = []
183 +
184 + audio_features_all = []
185 +
186 + # audio placeholder not dependent on audio_parts
187 + for audios in audios_list:
188 + if audios:
189 + audio_ph_list.append(
190 + [
191 + self.get_audio_placeholder(len(a), chunk_input, chunk_length)
192 + for a in audios
193 + ]
194 + )
195 + else:
196 + audio_ph_list.append([])
197 +
198 + for idx, audios in enumerate(audios_list):
199 + if audio_parts is not None:
200 + # same audio part merge
201 + audio_part = audio_parts[idx]
202 + merge_audio = []
203 + cur_audio = []
204 + for aid, (part, audio) in enumerate(zip(audio_part, audios)):
205 + if aid == 0 or audio_part[aid] == audio_part[aid - 1]:
206 + cur_audio.append(audio)
207 + else:
208 + merge_audio.append(np.hstack(cur_audio))
209 + cur_audio = [audio]
210 + if cur_audio:
211 + merge_audio.append(np.hstack(cur_audio))
212 +
213 + else:
214 + merge_audio = audios
215 +
216 + audio_feature_lens = []
217 +
218 + # If the audio exceeds 30 seconds, split it into chunks every 30 seconds.
219 + final_merge_audio = []
220 + max_audio_inp_len = 30 * (sampling_rate or 16000)
221 + for audio in merge_audio:
222 + if len(audio) <= max_audio_inp_len:
223 + final_merge_audio.append(audio)
224 + else:
225 + for i in range(math.ceil(len(audio) / max_audio_inp_len)):
226 + final_merge_audio.append(
227 + audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len]
228 + )
229 +
230 + if audios:
231 + audio_inputs = self.feature_extractor(
232 + final_merge_audio,
233 + sampling_rate=sampling_rate,
234 + return_attention_mask=True,
235 + padding="max_length",
236 + return_tensors="pt",
237 + **kwargs,
238 + )
239 + audio_feature = audio_inputs["input_features"]
240 + actual_lens = audio_inputs["attention_mask"].sum(dim=1)
241 +
242 + for feat, lens in zip(audio_feature, actual_lens):
243 + audio_features_all.append(feat[:, :lens])
244 + audio_feature_lens.append(lens)
245 +
246 + audio_feature_lens = torch.hstack(audio_feature_lens)
247 + audio_feature_lens_list.append(audio_feature_lens)
248 + else:
249 + audio_feature_lens_list.append([])
250 +
251 + if audio_features_all:
252 + audio_features = [i.permute(1, 0) for i in audio_features_all]
253 + audio_features = torch.nn.utils.rnn.pad_sequence(
254 + audio_features, batch_first=True, padding_value=0.0
255 + ).permute(0, 2, 1)
256 + else:
257 + audio_features = []
258 +
259 + return audio_features, audio_feature_lens_list, audio_ph_list
260 +
261 + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode
262 + # with CLIP->Llama
263 + def batch_decode(self, *args, **kwargs):
264 + """
265 + This method forwards all its arguments to LlamaTokenizerFast's
266 + [`~PreTrainedTokenizer.batch_decode`]. Please refer to the
267 + docstring of this method for more information.
268 + """
269 + output_ids = args[0]
270 + result_text = []
271 + for result in output_ids:
272 + result = result[result != 0]
273 + if len(result) > 0 and result[0] == self.tokenizer.bos_id:
274 + result = result[1:]
275 + if len(result) > 0 and result[-1] == self.tokenizer.eos_id:
276 + result = result[:-1]
277 + result_text.append(
278 + self.tokenizer.decode(result, *args[1:], **kwargs).strip()
279 + )
280 + return result_text
281 +
282 + # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode
283 + # with CLIP->Llama
284 + def decode(self, *args, **kwargs):
285 + """
286 + This method forwards all its arguments to LlamaTokenizerFast's
287 + [`~PreTrainedTokenizer.decode`]. Please refer to the docstring
288 + of this method for more information.
289 + """
290 + result = args[0]
291 + result = result[result != 0]
292 + if len(result) > 0 and result[0] == self.tokenizer.bos_id:
293 + result = result[1:]
294 + if len(result) > 0 and (
295 + result[-1] == self.tokenizer.eos_id
296 + or (
297 + hasattr(self.tokenizer, "eot_id")
298 + and result[-1] == self.tokenizer.eot_id
299 + )
300 + ):
301 + result = result[:-1]
302 + return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
303 +
304 + def _convert(self, input_str, max_inp_length: int | None = None, **kwargs):
305 + input_ids = self.tokenizer.encode(input_str, **kwargs)
306 + if max_inp_length is not None:
307 + input_ids = input_ids[:max_inp_length]
308 + input_ids = torch.tensor(input_ids, dtype=torch.int32)
309 +
310 + ## image bound
311 + start_cond = (input_ids == self.tokenizer.im_start_id) | (
312 + input_ids == self.tokenizer.slice_start_id
313 + )
314 + end_cond = (input_ids == self.tokenizer.im_end_id) | (
315 + input_ids == self.tokenizer.slice_end_id
316 + )
317 +
318 + image_start_idx = torch.where(start_cond)[0]
319 + image_start_idx += 1
320 + image_end_idx = torch.where(end_cond)[0]
321 +
322 + assert len(image_start_idx) == len(image_end_idx), (
323 + f"The number of image start tokens ({len(image_start_idx)}) "
324 + f"and end tokens ({len(image_end_idx)}) must match."
325 + )
326 +
327 + image_bounds = torch.hstack(
328 + [
329 + image_start_idx.unsqueeze(-1),
330 + image_end_idx.unsqueeze(-1),
331 + ]
332 + )
333 +
334 + ## audio bound
335 + audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0]
336 + audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0]
337 + assert len(audio_start_idx) == len(audio_end_idx)
338 + audio_bounds = torch.hstack(
339 + [(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)]
340 + )
341 +
342 + spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0]
343 + spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0]
344 + assert len(spk_start_idx) == len(spk_end_idx)
345 + spk_bounds = torch.hstack(
346 + [(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)]
347 + )
348 +
349 + return input_ids, image_bounds, audio_bounds, spk_bounds
350 +
351 + def _convert_omni_to_inputs(
352 + self,
353 + images,
354 + audio_phs,
355 + texts: str | list[str],
356 + truncation=None,
357 + max_length=None,
358 + max_slice_nums=None,
359 + use_image_id=None,
360 + return_tensors=None,
361 + **kwargs,
362 + ):
363 + if images is None and audio_phs is None:
364 + model_inputs = self.tokenizer(
365 + texts,
366 + return_tensors=return_tensors,
367 + truncation=truncation,
368 + max_length=max_length,
369 + **kwargs,
370 + )
371 + return MiniCPMOBatchFeature(data={**model_inputs})
372 +
373 + image_tag = "(<image>./</image>)"
374 + image_pattern = r"\(<image>./</image>\)"
375 + audio_tag = "(<audio>./</audio>)"
376 + audio_pattern = r"\(<audio>./</audio>\)"
377 + split_pattern = rf"({image_pattern}|{audio_pattern})"
378 +
379 + if isinstance(texts, str):
380 + texts = [texts]
381 +
382 + bs = len(texts)
383 + if images is not None:
384 + images, image_sizes, tgt_sizes = (
385 + images["pixel_values"],
386 + images["image_sizes"],
387 + images["tgt_sizes"],
388 + )
389 + else:
390 + images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs
391 +
392 + input_ids_list = []
393 + image_bounds_list = []
394 + audio_bounds_list = []
395 + spk_bounds_list = []
396 +
397 + for index, text in enumerate(texts):
398 + text_chunks = regex.split(split_pattern, text)
399 +
400 + image_tags = regex.findall(image_pattern, text)
401 + audio_tags = regex.findall(audio_pattern, text)
402 +
403 + if image_tags:
404 + assert images is not None
405 + assert len(image_tags) == len(image_sizes[index])
406 + if audio_tags:
407 + assert audio_phs is not None
408 + assert len(audio_tags) == len(audio_phs[index])
409 +
410 + image_id = 0
411 + audio_id = 0
412 + for i, chunk in enumerate(text_chunks):
413 + if chunk == image_tag:
414 + image_placeholder = (
415 + self.image_processor.get_slice_image_placeholder(
416 + image_sizes[index][image_id],
417 + image_id,
418 + max_slice_nums,
419 + use_image_id,
420 + )
421 + )
422 + image_id += 1
423 + text_chunks[i] = image_placeholder
424 + elif chunk == audio_tag:
425 + audio_placeholder = audio_phs[index][audio_id]
426 + audio_id += 1
427 + text_chunks[i] = audio_placeholder
428 +
429 + final_text = "".join(text_chunks)
430 + input_ids, image_bounds, audio_bounds, spk_bounds = self._convert(
431 + final_text, max_length, **kwargs
432 + )
433 +
434 + input_ids_list.append(input_ids)
435 + image_bounds_list.append(image_bounds)
436 + audio_bounds_list.append(audio_bounds)
437 + spk_bounds_list.append(spk_bounds)
438 +
439 + padded_input_ids, padding_lengths = self.pad(
440 + input_ids_list, padding_side="left"
441 + )
442 + attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool)
443 + for i, length in enumerate(padding_lengths):
444 + image_bounds_list[i] = image_bounds_list[i] + length
445 + audio_bounds_list[i] = audio_bounds_list[i] + length
446 + spk_bounds_list[i] = spk_bounds_list[i] + length
447 + attention_mask[i, :length] = False
448 +
449 + data = {
450 + "input_ids": padded_input_ids,
451 + "attention_mask": attention_mask,
452 + "pixel_values": images,
453 + "image_sizes": image_sizes,
454 + "image_bound": image_bounds_list,
455 + "tgt_sizes": tgt_sizes,
456 + "audio_bounds": audio_bounds_list,
457 + "spk_bounds": spk_bounds_list,
458 + }
459 +
460 + return data
461 +
462 + @property
463 + # Copied from
464 + # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
465 + def model_input_names(self):
466 + tokenizer_input_names = self.tokenizer.model_input_names
467 + image_processor_input_names = self.image_processor.model_input_names
468 + feature_extractor_input_names = self.feature_extractor.model_input_names
469 + return list(
470 + dict.fromkeys(
471 + tokenizer_input_names
472 + + image_processor_input_names
473 + + feature_extractor_input_names
474 + )
475 + )
476 +
477 + def pad(
478 + self,
479 + inputs,
480 + max_length=None,
481 + padding_value=0,
482 + padding_side="left",
483 + ):
484 + if not inputs:
485 + return torch.empty(0), []
486 +
487 + items = []
488 + if isinstance(inputs[0], list):
489 + assert isinstance(inputs[0][0], torch.Tensor)
490 + for it in inputs:
491 + for tr in it:
492 + items.append(tr)
493 + else:
494 + assert isinstance(inputs[0], torch.Tensor)
495 + items = inputs
496 +
497 + batch_size = len(items)
498 + shape = items[0].shape
499 + dim = len(shape)
500 + assert dim <= 2
501 + if max_length is None:
502 + max_length = 0
503 + max_length = max(max_length, max(item.shape[-1] for item in items))
504 + min_length = min(item.shape[-1] for item in items)
505 + dtype = items[0].dtype
506 +
507 + if dim == 0:
508 + return torch.stack([item for item in items], dim=0), [0]
509 + elif dim == 1:
510 + if max_length == min_length:
511 + return (
512 + torch.stack([item for item in items], dim=0),
513 + [0] * batch_size,
514 + )
515 + tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
516 + else:
517 + tensor = (
518 + torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
519 + + padding_value
520 + )
521 +
522 + padding_length = []
523 + for i, item in enumerate(items):
524 + if dim == 1:
525 + if padding_side == "left":
526 + tensor[i, -len(item) :] = item.clone()
527 + else:
528 + tensor[i, : len(item)] = item.clone()
529 + elif dim == 2:
530 + if padding_side == "left":
531 + tensor[i, -len(item) :, :] = item.clone()
532 + else:
533 + tensor[i, : len(item), :] = item.clone()
534 + padding_length.append(tensor.shape[-1] - len(item))
535 +
536 + return tensor, padding_length
537 +
538 +
539 +class MelSpectrogramFeatures(torch.nn.Module):
540 + def __init__(
541 + self,
542 + sample_rate=24000,
543 + n_fft=1024,
544 + hop_length=256,
545 + n_mels=100,
546 + padding: Literal["center", "same"] = "center",
547 + ):
548 + super().__init__()
549 + if padding not in ["center", "same"]:
550 + raise ValueError("Padding must be 'center' or 'same'.")
551 + self.padding = padding
552 + self.mel_spec = torchaudio.transforms.MelSpectrogram(
553 + sample_rate=sample_rate,
554 + n_fft=n_fft,
555 + hop_length=hop_length,
556 + n_mels=n_mels,
557 + center=padding == "center",
558 + power=1,
559 + )
560 +
561 + def __call__(self, audio: torch.Tensor) -> torch.Tensor:
562 + """
563 + audio: Tensor([num_channels, num_samples])
564 + """
565 + return super().__call__(audio)
566 +
567 + def forward(self, audio: torch.Tensor) -> torch.Tensor:
568 + """
569 + audio: Tensor([num_channels, num_samples])
570 + """
571 + mel: torch.Tensor = self.mel_spec(audio)
572 + features = torch.log(torch.clip(mel, min=1e-5))
573 + return features
574 +
575 +
576 +class ChatTTSProcessor:
577 + def __init__(self, text_tokenizer):
578 + self.audio_processor = MelSpectrogramFeatures()
579 + self.text_tokenizer = text_tokenizer
580 +
581 + def __call__(self, text_list, audio_list):
582 + assert len(text_list) == len(audio_list)
583 + input_ids_varlen = []
584 + for text in text_list:
585 + input_ids_ = self.text_tokenizer.encode(
586 + text, return_tensors="pt", add_special_tokens=False
587 + ) # [1, seq_len]
588 + input_ids_ = input_ids_.squeeze(0) # [seq_len]
589 + input_ids_varlen.append(input_ids_)
590 +
591 + audio_features_varlen = []
592 + for audio in audio_list:
593 + assert audio.shape.__len__() == 1 # [seq_len]
594 + try:
595 + mel = self.audio_processor(audio) # [100(num_mel_bins), seq_len_mel]
596 + except Exception as e:
597 + raise e
598 + audio_features_varlen.append(mel)
599 +
600 + return {
601 + "tts_input_ids_varlen": input_ids_varlen, # return List[Tensor]
602 + "tts_input_features_varlen": audio_features_varlen, # return List[Tensor]
603 + }
vllm/transformers_utils/processors/minicpmv.py new
+314
@@ -0,0 +1,314 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Copyright 2024 The HuggingFace Inc. team.
4 +#
5 +# Licensed under the Apache License, Version 2.0 (the "License");
6 +# you may not use this file except in compliance with the License.
7 +# You may obtain a copy of the License at
8 +#
9 +# http://www.apache.org/licenses/LICENSE-2.0
10 +#
11 +# Unless required by applicable law or agreed to in writing, software
12 +# distributed under the License is distributed on an "AS IS" BASIS,
13 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 +# See the License for the specific language governing permissions and
15 +# limitations under the License.
16 +"""
17 +Processor class for MiniCPMV.
18 +"""
19 +
20 +from typing import TypeAlias
21 +
22 +import regex
23 +import torch
24 +from transformers.image_processing_utils import BatchFeature
25 +from transformers.image_utils import ImageInput
26 +from transformers.processing_utils import ProcessorMixin
27 +from transformers.tokenization_utils_base import (
28 + PaddingStrategy,
29 + PreTokenizedInput,
30 + TextInput,
31 + TruncationStrategy,
32 +)
33 +from transformers.utils import TensorType
34 +
35 +MiniCPMVBatchFeature: TypeAlias = BatchFeature
36 +
37 +
38 +class MiniCPMVProcessor(ProcessorMixin):
39 + r"""
40 + Constructs a MiniCPMV processor which wraps a MiniCPMV image
41 + processor and a MiniCPMV tokenizer into a single processor.
42 +
43 + [`MiniCPMVProcessor`] offers all the functionalities of
44 + [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the
45 + [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`]
46 + for more information.
47 +
48 + Args:
49 + image_processor ([`MiniCPMVImageProcessor`], *optional*):
50 + The image processor is a required input.
51 + tokenizer ([`LlamaTokenizerWrapper`], *optional*):
52 + The tokenizer is a required input.
53 + """
54 +
55 + attributes = ["image_processor", "tokenizer"]
56 + image_processor_class = "AutoImageProcessor"
57 + tokenizer_class = "AutoTokenizer"
58 +
59 + def __init__(self, image_processor=None, tokenizer=None):
60 + super().__init__(image_processor, tokenizer)
61 + self.version = image_processor.version
62 +
63 + def __call__(
64 + self,
65 + text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],
66 + images: ImageInput = None,
67 + padding: bool | str | PaddingStrategy = False,
68 + truncation: bool | str | TruncationStrategy = None,
69 + max_length: int | None = None,
70 + do_pad: bool | None = True,
71 + return_tensors: str | TensorType | None = TensorType.PYTORCH,
72 + ) -> MiniCPMVBatchFeature:
73 + """Run the vendored MiniCPMV processor on a (text, images) pair.
74 +
75 + Only single-sample input is currently supported; batched input is
76 + coming soon. ``images`` is forwarded to the underlying image
77 + processor and ``text`` is tokenized with image placeholders
78 + replaced by the appropriate slice tokens. Returns a
79 + ``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when
80 + images are provided) ``pixel_values``, ``image_sizes``,
81 + ``image_bound`` and ``tgt_sizes``.
82 + """
83 + if images is not None:
84 + image_inputs = self.image_processor(
85 + images, do_pad=do_pad, return_tensors=return_tensors
86 + )
87 + else:
88 + image_inputs = {}
89 + return self._convert_images_texts_to_inputs(
90 + image_inputs, text, max_length=max_length
91 + )
92 +
93 + # Copied from transformers.models.clip.processing_clip.CLIPProcessor
94 + # .batch_decode with CLIP->Llama
95 + def batch_decode(self, *args, **kwargs):
96 + """
97 + This method forwards all its arguments to LlamaTokenizerFast's
98 + [`~PreTrainedTokenizer.batch_decode`]. Please refer to the
99 + docstring of this method for more information.
100 + """
101 + output_ids = args[0]
102 + result_text = []
103 +
104 + bos_id = getattr(
105 + self.tokenizer,
106 + "bos_token_id",
107 + getattr(self.tokenizer, "bos_id", 1),
108 + )
109 + eos_id = getattr(
110 + self.tokenizer,
111 + "eos_token_id",
112 + getattr(self.tokenizer, "eos_id", 2),
113 + )
114 +
115 + for result in output_ids:
116 + result = result[result != 0]
117 + if len(result) > 0 and result[0] == bos_id:
118 + result = result[1:]
119 + if len(result) > 0 and result[-1] == eos_id:
120 + result = result[:-1]
121 + result_text.append(
122 + self.tokenizer.decode(result, *args[1:], **kwargs).strip()
123 + )
124 + return result_text
125 +
126 + # Copied from transformers.models.clip.processing_clip.CLIPProcessor
127 + # .decode with CLIP->Llama
128 + def decode(self, *args, **kwargs):
129 + """
130 + This method forwards all its arguments to LlamaTokenizerFast's
131 + [`~PreTrainedTokenizer.decode`]. Please refer to the docstring
132 + of this method for more information.
133 + """
134 + result = args[0]
135 + result = result[result != 0]
136 +
137 + bos_id = getattr(
138 + self.tokenizer,
139 + "bos_token_id",
140 + getattr(self.tokenizer, "bos_id", 1),
141 + )
142 + eos_id = getattr(
143 + self.tokenizer,
144 + "eos_token_id",
145 + getattr(self.tokenizer, "eos_id", 2),
146 + )
147 + eot_id = getattr(self.tokenizer, "eot_id", None)
148 +
149 + if len(result) > 0 and result[0] == bos_id:
150 + result = result[1:]
151 + if len(result) > 0 and (
152 + result[-1] == eos_id or (eot_id is not None and result[-1] == eot_id)
153 + ):
154 + result = result[:-1]
155 + return self.tokenizer.decode(result, *args[1:], **kwargs).strip()
156 +
157 + def _convert(self, input_str, max_inp_length: int | None = None):
158 + add_bos = getattr(self.tokenizer, "add_bos_token", False)
159 + if self.version == 2.5 or add_bos:
160 + input_ids = self.tokenizer.encode(input_str)
161 + else:
162 + bos_id = getattr(
163 + self.tokenizer,
164 + "bos_token_id",
165 + getattr(self.tokenizer, "bos_id", 1),
166 + )
167 + input_ids = [bos_id] + self.tokenizer.encode(input_str)
168 +
169 + if max_inp_length is not None:
170 + input_ids = input_ids[:max_inp_length]
171 + input_ids = torch.tensor(input_ids, dtype=torch.int32)
172 +
173 + im_start_id = getattr(
174 + self.tokenizer,
175 + "im_start_id",
176 + self.tokenizer.convert_tokens_to_ids("<im_start>"),
177 + )
178 + im_end_id = getattr(
179 + self.tokenizer,
180 + "im_end_id",
181 + self.tokenizer.convert_tokens_to_ids("<im_end>"),
182 + )
183 +
184 + image_start_tokens = torch.where(input_ids == im_start_id)[0]
185 + image_start_tokens += 1
186 + image_end_tokens = torch.where(input_ids == im_end_id)[0]
187 + assert len(image_start_tokens) == len(image_end_tokens), (
188 + f"The number of image start tokens ({len(image_start_tokens)}) "
189 + f"and end tokens ({len(image_end_tokens)}) must match."
190 + )
191 + image_bounds = torch.hstack(
192 + [
193 + image_start_tokens.unsqueeze(-1),
194 + image_end_tokens.unsqueeze(-1),
195 + ]
196 + )
197 + return input_ids.unsqueeze(0), image_bounds
198 +
199 + def _convert_images_texts_to_inputs(
200 + self,
201 + images,
202 + texts,
203 + do_pad=False,
204 + truncation=None,
205 + max_length=None,
206 + return_tensors=None,
207 + ):
208 + if not len(images):
209 + model_inputs = self.tokenizer(
210 + texts,
211 + return_tensors=return_tensors,
212 + padding=do_pad,
213 + truncation=truncation,
214 + max_length=max_length,
215 + )
216 + return MiniCPMVBatchFeature(data={**model_inputs})
217 +
218 + pattern = "(<image>./</image>)"
219 + images_val = images["pixel_values"]
220 + image_sizes = images["image_sizes"]
221 + tgt_sizes = images["tgt_sizes"]
222 +
223 + image_tags = regex.findall(pattern, texts)
224 + assert len(image_tags) == len(image_sizes[0])
225 + text_chunks = texts.split(pattern)
226 + final_texts = ""
227 + for i in range(len(image_tags)):
228 + placeholder = self.image_processor.get_slice_image_placeholder(
229 + image_sizes[0][i]
230 + )
231 + final_texts = final_texts + text_chunks[i] + placeholder
232 + final_texts += text_chunks[-1]
233 + input_ids, image_bounds = self._convert(final_texts, max_length)
234 + return MiniCPMVBatchFeature(
235 + data={
236 + "input_ids": input_ids,
237 + "pixel_values": images_val,
238 + "image_sizes": image_sizes,
239 + "image_bound": [image_bounds],
240 + "tgt_sizes": tgt_sizes,
241 + }
242 + )
243 +
244 + @property
245 + # Copied from
246 + # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
247 + def model_input_names(self):
248 + tokenizer_input_names = self.tokenizer.model_input_names
249 + image_processor_input_names = self.image_processor.model_input_names
250 + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
251 +
252 + def pad(
253 + self,
254 + orig_items,
255 + key,
256 + max_length=None,
257 + padding_value=0,
258 + padding_side="left",
259 + ):
260 + if not orig_items:
261 + return torch.empty(0)
262 +
263 + items = []
264 + if isinstance(orig_items[0][key], list):
265 + assert isinstance(orig_items[0][key][0], torch.Tensor)
266 + for it in orig_items:
267 + for tr in it[key]:
268 + items.append({key: tr})
269 + else:
270 + assert isinstance(orig_items[0][key], torch.Tensor)
271 + items = orig_items
272 +
273 + batch_size = len(items)
274 + shape = items[0][key].shape
275 + dim = len(shape)
276 + assert dim <= 3
277 + if max_length is None:
278 + max_length = 0
279 + max_length = max(max_length, max(item[key].shape[-1] for item in items))
280 + min_length = min(item[key].shape[-1] for item in items)
281 + dtype = items[0][key].dtype
282 +
283 + if dim == 1:
284 + return torch.cat([item[key] for item in items], dim=0)
285 + elif dim == 2:
286 + if max_length == min_length:
287 + return torch.cat([item[key] for item in items], dim=0)
288 + tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
289 + else:
290 + tensor = (
291 + torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype)
292 + + padding_value
293 + )
294 +
295 + for i, item in enumerate(items):
296 + tensor_to_pad = item[key]
297 + if tensor_to_pad.shape[0] != 1:
298 + raise ValueError(
299 + f"Expected leading batch size of 1 for padding, "
300 + f"but got shape {tensor_to_pad.shape}"
301 + )
302 + squeezed = tensor_to_pad.squeeze(0)
303 + if dim == 2:
304 + if padding_side == "left":
305 + tensor[i, -squeezed.shape[0] :] = squeezed.clone()
306 + else:
307 + tensor[i, : squeezed.shape[0]] = squeezed.clone()
308 + elif dim == 3:
309 + if padding_side == "left":
310 + tensor[i, -squeezed.shape[0] :, :] = squeezed.clone()
311 + else:
312 + tensor[i, : squeezed.shape[0], :] = squeezed.clone()
313 +
314 + return tensor