A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[Misc] Clean up renderers (#36770)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 12, 2026 at 00:39 UTC
196802dfa68c512b5360546003b2a35259de66da
12 files changed
+136
-220
tests/models/multimodal/processing/test_common.py
+25
-68
@@ -6,9 +6,6 @@ from functools import partial
6
7
import numpy as np
8
import pytest
9
-from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
10
-from mistral_common.protocol.instruct.messages import UserMessage
11
-from mistral_common.protocol.instruct.request import ChatCompletionRequest
9
from PIL import Image
10
11
from vllm.config import ModelConfig
@@ -21,7 +18,10 @@ from vllm.config.multimodal import (
18
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
19
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
20
from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
24
-from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
21
+from vllm.multimodal.processing import (
22
+ BaseMultiModalProcessor,
23
+ InputProcessingContext,
24
+)
25
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
26
from vllm.utils.mistral import is_mistral_tokenizer
27
@@ -74,20 +74,6 @@ def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
74
return mm_data
75
76
77
-# For some multimodal models, tokenizer will always add bos_token
78
-# at the beginning of prompt by default, causing hf_processor outputs
79
-# incorrect token ids. So we need use `add_special_tokens=False` here
80
-# to leave bos_token to be added by the processor.
81
-_ADD_SPECIAL_TOKENS_OVERRIDES = {
82
- "lfm2_vl": False,
83
- "nemotron_parse": False,
84
- "ovis": False,
85
- "ovis2_5": False,
86
- "paligemma": False,
87
- "ultravox": False,
88
- "whisper": False,
89
-}
90
-
77
_IGNORE_MM_KEYS = {
78
# In Ultravox, the audio_features can be different depending on padding
79
# The slight difference should not be a problem though, since
@@ -152,63 +138,34 @@ def get_text_token_prompts(
138
parsed_data = processor.info.parse_mm_data(mm_data)
139
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
140
155
- text_prompt: str | None
156
- token_prompt: list[int]
141
if is_mistral_tokenizer(tokenizer):
158
- # ChatCompletionRequest only supports ImageChunk natively;
159
- # for other modalities (e.g. audio), fall back to the model's
160
- # own dummy inputs builder which knows the right placeholders.
161
- has_non_image = any(
162
- k != "image" and count > 0 for k, count in mm_counts.items()
142
+ inputs = dummy_inputs.get_dummy_processor_inputs(
143
+ model_config.max_model_len,
144
+ mm_counts,
145
+ mm_options={},
146
+ # Assume all Mistral models define this extra argument
147
+ mm_data=mm_data, # type: ignore[call-arg]
148
)
164
-
165
- if has_non_image:
166
- inputs = dummy_inputs.get_dummy_processor_inputs(
167
- model_config.max_model_len,
168
- mm_counts,
169
- mm_options={},
170
- )
171
- text_prompt = None
172
- token_prompt = (
173
- inputs.prompt
174
- if isinstance(inputs.prompt, list)
175
- else tokenizer.encode(inputs.prompt, add_special_tokens=False)
176
- )
177
- else:
178
- images = parsed_data.get("image", [])
179
- request = ChatCompletionRequest(
180
- messages=[
181
- UserMessage(
182
- content=[
183
- TextChunk(text=""),
184
- *(ImageChunk(image=image) for image in images),
185
- ]
186
- ),
187
- ]
188
- )
189
- res = tokenizer.mistral.encode_chat_completion(request)
190
-
191
- # Mistral does not support decode_tokens with
192
- # skip_special_tokens=False
193
- text_prompt = None
194
- token_prompt = res.tokens
149
else:
150
inputs = dummy_inputs.get_dummy_processor_inputs(
151
model_config.max_model_len,
152
mm_counts,
153
mm_options={},
154
)
201
- # Some models (e.g., Kimi-Audio) return token IDs directly instead of str
202
- if isinstance(inputs.prompt, list):
203
- text_prompt = None
204
- token_prompt = inputs.prompt
205
- else:
206
- assert isinstance(inputs.prompt, str)
207
- text_prompt = inputs.prompt
208
- token_prompt = tokenizer.encode(
209
- text_prompt,
210
- add_special_tokens=_ADD_SPECIAL_TOKENS_OVERRIDES.get(model_type, True),
211
- )
155
+
156
+ text_prompt: str | None
157
+ token_prompt: list[int]
158
+ if isinstance(inputs.prompt, list):
159
+ text_prompt = None
160
+ token_prompt = inputs.prompt
161
+ elif isinstance(inputs.prompt, str):
162
+ text_prompt = inputs.prompt
163
+ token_prompt = tokenizer.encode(
164
+ text_prompt,
165
+ **processor.info.get_default_tok_params().get_encode_kwargs(),
166
+ )
167
+ else:
168
+ raise TypeError(type(inputs.prompt))
169
170
return text_prompt, token_prompt
171
@@ -448,7 +405,7 @@ def test_processing_correctness(
405
)
406
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
407
pytest.skip(
451
- "Voxtral Realtime doesn't make use of any place-holder"
408
+ "Voxtral Realtime doesn't make use of any place-holder "
409
"tokens and hence cannot pass the processing "
410
"correctness test as is. Let's revisit adapting this "
411
"test once more realtime models exist."
vllm/config/model.py
+16
@@ -532,6 +532,22 @@ class ModelConfig:
532
self._architecture = arch
533
logger.info("Resolved architecture: %s", arch)
534
535
+ # Set default tokenizer modes based on model architecture
536
+ if self.tokenizer_mode == "auto":
537
+ if arch == "Grok1ForCausalLM":
538
+ self.tokenizer_mode = "grok2"
539
+ elif arch == "MoonshotKimiaForCausalLM":
540
+ self.tokenizer_mode = "kimi_audio"
541
+ elif arch == "QwenVLForConditionalGeneration":
542
+ self.tokenizer_mode = "qwen_vl"
543
+
544
+ if self.tokenizer_mode != "auto":
545
+ logger.info(
546
+ "Defaulting to tokenizer_mode=%r for %s",
547
+ self.tokenizer_mode,
548
+ arch,
549
+ )
550
+
551
# Init pooler config if needed
552
if self.runner_type == "pooling":
553
if self.pooler_config is None:
vllm/model_executor/models/kimi_audio.py
+47
-35
@@ -10,11 +10,13 @@ from typing import Any, ClassVar, Literal
10
import numpy as np
11
import torch
12
import torch.nn as nn
13
+from huggingface_hub import snapshot_download
14
from safetensors import safe_open
15
from transformers import BatchFeature
16
from transformers import WhisperConfig as HFWhisperConfig
17
18
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
19
+from vllm.config.multimodal import BaseDummyOptions
20
from vllm.inputs.data import PromptType, TokensPrompt
21
from vllm.model_executor.layers.logits_processor import LogitsProcessor
22
from vllm.model_executor.model_loader.weight_utils import (
@@ -47,7 +49,10 @@ from vllm.multimodal.processing import (
49
BaseProcessingInfo,
50
PromptReplacement,
51
)
50
-from vllm.multimodal.processing.processor import BaseMultiModalProcessor
52
+from vllm.multimodal.processing.processor import (
53
+ BaseMultiModalProcessor,
54
+ ProcessorInputs,
55
+)
56
from vllm.sequence import IntermediateTensors
57
from vllm.tokenizers import cached_get_tokenizer
58
from vllm.tokenizers.kimi_audio import KimiAudioTokenizer
@@ -59,6 +64,15 @@ from vllm.v1.sample.metadata import SamplingMetadata
64
KIMIA_WHISPER_SUBFOLDER = "whisper-large-v3"
65
66
67
+def _get_whisper_local_path(repo_id: str):
68
+ if os.path.exists(repo_id):
69
+ repo_local_path = repo_id
70
+ else:
71
+ repo_local_path = snapshot_download(repo_id, local_files_only=True)
72
+
73
+ return os.path.join(repo_local_path, KIMIA_WHISPER_SUBFOLDER)
74
+
75
+
76
def _get_feat_extract_output_lengths(input_lengths: torch.Tensor) -> torch.Tensor:
77
"""Compute output lengths after Whisper feature extraction.
78
@@ -88,10 +102,10 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
102
# Load Whisper config from subfolder (authoritative source)
103
# Kimi-Audio stores Whisper config in whisper-large-v3/config.json
104
model_path = vllm_config.model_config.model
91
- whisper_config_path = os.path.join(model_path, KIMIA_WHISPER_SUBFOLDER)
105
106
# Load WhisperConfig from the subfolder
94
- whisper_config = HFWhisperConfig.from_pretrained(whisper_config_path)
107
+ whisper_dir = _get_whisper_local_path(model_path)
108
+ whisper_config = HFWhisperConfig.from_pretrained(whisper_dir)
109
110
# Temporarily replace hf_config for WhisperEncoder.__init__()
111
original_config = vllm_config.model_config.hf_config
@@ -114,28 +128,18 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
128
class KimiAudioProcessingInfo(BaseProcessingInfo):
129
"""Processing info for vLLM registry."""
130
117
- def get_hf_config(self):
118
- return self.ctx.model_config.hf_config
119
-
131
def get_hf_processor(self, **kwargs: object) -> KimiAudioProcessor:
121
- """Get KimiAudioProcessor with feature extractor and tokenizer."""
122
- # Use vLLM's cached loader for feature extractor
132
feature_extractor = cached_feature_extractor_from_config(
133
self.ctx.model_config,
134
subfolder=KIMIA_WHISPER_SUBFOLDER,
135
)
136
128
- # Use vLLM's standard tokenizer loading (respects tokenizer_mode)
129
- tokenizer = self.get_tokenizer()
130
-
131
- # Construct processor directly
137
return KimiAudioProcessor(
138
feature_extractor=feature_extractor,
134
- tokenizer=tokenizer,
139
+ tokenizer=self.get_tokenizer(),
140
)
141
142
def get_feature_extractor(self, **kwargs: object):
138
- """Get feature extractor using vLLM's cached loader."""
143
return cached_feature_extractor_from_config(
144
self.ctx.model_config, subfolder=KIMIA_WHISPER_SUBFOLDER
145
)
@@ -144,26 +148,16 @@ class KimiAudioProcessingInfo(BaseProcessingInfo):
148
return {"audio": 1}
149
150
def get_data_parser(self) -> "KimiAudioMultiModalDataParser":
147
- """Get data parser for audio inputs."""
151
+ feature_extractor = self.get_feature_extractor()
152
return KimiAudioMultiModalDataParser(
153
+ target_sr=feature_extractor.sampling_rate,
154
expected_hidden_size=self._get_expected_hidden_size(),
155
)
156
157
158
class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo]):
154
- """Dummy inputs builder for vLLM registry."""
155
-
156
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> list[int]:
157
- """Return dummy text as token IDs directly."""
158
- num_audios = mm_counts.get("audio", 0)
159
- if num_audios == 0:
160
- return [198] # "Transcribe" tokenized
161
- # Return as token IDs directly to avoid tokenizer issues
162
- return [
163
- KimiAudioProcessor.KIMIA_MEDIA_BEGIN,
164
- KimiAudioProcessor.KIMIA_TEXT_BLANK,
165
- KimiAudioProcessor.KIMIA_MEDIA_END,
166
- ] * num_audios
159
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
160
+ return ""
161
162
def get_dummy_mm_data(
163
self,
@@ -186,6 +180,29 @@ class KimiAudioDummyInputsBuilder(BaseDummyInputsBuilder[KimiAudioProcessingInfo
180
),
181
}
182
183
+ def get_dummy_processor_inputs(
184
+ self,
185
+ seq_len: int,
186
+ mm_counts: Mapping[str, int],
187
+ mm_options: Mapping[str, BaseDummyOptions],
188
+ ) -> ProcessorInputs:
189
+ dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
190
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
191
+
192
+ num_audios = mm_counts.get("audio", 0)
193
+ dummy_tokens = (
194
+ [198]
195
+ if num_audios == 0
196
+ else [
197
+ KimiAudioProcessor.KIMIA_MEDIA_BEGIN,
198
+ KimiAudioProcessor.KIMIA_TEXT_BLANK,
199
+ KimiAudioProcessor.KIMIA_MEDIA_END,
200
+ ]
201
+ * num_audios
202
+ )
203
+
204
+ return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
205
+
206
207
# Field config for Kimi-Audio multimodal data
208
_KIMIAUDIO_FIELD_CONFIG = {
@@ -197,10 +214,6 @@ _KIMIAUDIO_FIELD_CONFIG = {
214
class KimiAudioMultiModalDataParser(MultiModalDataParser):
215
"""Custom data parser for Kimi-Audio multimodal data."""
216
200
- def __init__(self, **kwargs):
201
- # Whisper expects 16kHz audio
202
- super().__init__(target_sr=16000, **kwargs)
203
-
217
def _parse_audio_data(
218
self,
219
data: dict[str, torch.Tensor] | ModalityData[AudioItem],
@@ -589,9 +602,8 @@ class KimiAudioForConditionalGeneration(
602
loaded = loader.load_weights(main_weights, mapper=self.hf_to_vllm_mapper)
603
604
# Load Whisper encoder weights from subfolder
592
- whisper_path = os.path.join(
593
- self.model_path, f"{KIMIA_WHISPER_SUBFOLDER}/model.safetensors"
594
- )
605
+ whisper_dir = _get_whisper_local_path(self.model_path)
606
+ whisper_path = os.path.join(whisper_dir, "model.safetensors")
607
if os.path.exists(whisper_path):
608
whisper_loaded = self._load_whisper_weights_from_file(whisper_path)
609
loaded.update(whisper_loaded)
vllm/model_executor/models/mllama4.py
-12
@@ -63,12 +63,10 @@ from vllm.multimodal.processing import (
63
BaseDummyInputsBuilder,
64
BaseMultiModalProcessor,
65
BaseProcessingInfo,
66
- InputProcessingContext,
66
PromptReplacement,
67
PromptUpdate,
68
PromptUpdateDetails,
69
)
71
-from vllm.renderers import TokenizeParams
70
from vllm.sequence import IntermediateTensors
71
from vllm.utils.tensor_schema import TensorSchema, TensorShape
72
@@ -546,9 +544,6 @@ class Llama4VisionModel(nn.Module):
544
545
546
class Mllama4ProcessingInfo(BaseProcessingInfo):
549
- def __init__(self, ctx: InputProcessingContext) -> None:
550
- super().__init__(ctx)
551
-
547
def get_hf_config(self) -> Llama4Config:
548
return self.ctx.get_hf_config(Llama4Config)
549
@@ -557,9 +552,6 @@ class Mllama4ProcessingInfo(BaseProcessingInfo):
552
Llama4Processor, use_fast=kwargs.pop("use_fast", True), **kwargs
553
)
554
560
- def get_default_tok_params(self) -> TokenizeParams:
561
- return super().get_default_tok_params().with_kwargs(add_special_tokens=False)
562
-
555
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
556
# Although vLLM can support more images from an infra capability
557
# perspective, we do not recommend using >10 images in practice.
@@ -597,10 +589,6 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
589
mm_kwargs: Mapping[str, object],
590
tok_kwargs: Mapping[str, object],
591
) -> BatchFeature:
600
- tokenizer = self.info.get_tokenizer()
601
-
602
- if mm_data is None:
603
- return tokenizer(prompt, add_special_tokens=False) # exclude bos
592
processed_outputs = super()._call_hf_processor(
593
prompt=prompt,
594
mm_data=mm_data,
vllm/model_executor/models/pixtral.py
+10
-4
@@ -172,12 +172,20 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
172
seq_len: int,
173
mm_counts: Mapping[str, int],
174
mm_options: Mapping[str, BaseDummyOptions],
175
+ mm_data: MultiModalDataDict | None = None,
176
) -> ProcessorInputs:
177
tokenizer = self.info.get_tokenizer()
178
179
dummy_text = self.get_dummy_text(mm_counts)
179
- dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
180
- dummy_images = dummy_mm_data.get("image", [])
180
+ dummy_mm_data = (
181
+ self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
182
+ if mm_data is None
183
+ else mm_data
184
+ )
185
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
186
+ dummy_images = (
187
+ [] if "image" not in dummy_mm_data else dummy_mm_items["image"].get_all()
188
+ )
189
190
request = ChatCompletionRequest(
191
messages=[
@@ -192,8 +200,6 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
200
res = tokenizer.mistral.encode_chat_completion(request)
201
dummy_tokens = res.tokens
202
195
- dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
196
-
203
return ProcessorInputs(prompt=dummy_tokens, mm_data_items=dummy_mm_items)
204
205
vllm/model_executor/models/voxtral.py
+10
-2
@@ -150,13 +150,21 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
150
seq_len: int,
151
mm_counts: Mapping[str, int],
152
mm_options: Mapping[str, BaseDummyOptions],
153
+ mm_data: MultiModalDataDict | None = None,
154
) -> ProcessorInputs:
155
tokenizer = self.info.get_tokenizer()
156
feature_extractor = self.info.get_hf_processor().feature_extractor
157
158
dummy_text = self.get_dummy_text(mm_counts)
158
- dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
159
- dummy_audios = dummy_mm_data.get("audio", [])
159
+ dummy_mm_data = (
160
+ self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
161
+ if mm_data is None
162
+ else mm_data
163
+ )
164
+ dummy_mm_items = self.info.parse_mm_data(dummy_mm_data)
165
+ dummy_audios = (
166
+ [] if "audio" not in dummy_mm_data else dummy_mm_items["audio"].get_all()
167
+ )
168
169
audio_chunks: list[AudioChunk] = []
170
format = "wav"
vllm/renderers/qwen_vl.py
+1
-2
@@ -6,11 +6,10 @@ from vllm.config import VllmConfig
6
from vllm.tokenizers import cached_get_tokenizer
7
from vllm.tokenizers.qwen_vl import QwenVLTokenizer
8
9
-from .base import BaseRenderer
9
from .hf import HfRenderer
10
11
13
-class QwenVLRenderer(BaseRenderer[QwenVLTokenizer]):
12
+class QwenVLRenderer(HfRenderer):
13
@classmethod
14
def from_config( # type: ignore[override]
15
cls,
vllm/renderers/registry.py
-7
@@ -80,13 +80,6 @@ def renderer_from_config(config: "VllmConfig", **kwargs):
80
model_config, **kwargs
81
)
82
83
- # Override tokenizer_mode for Kimi-Audio models
84
- if model_config.architecture == "MoonshotKimiaForCausalLM":
85
- tokenizer_mode = "kimi_audio"
86
- # Update model_config so other components (e.g., multimodal registry)
87
- # also use the correct tokenizer mode
88
- model_config.tokenizer_mode = "kimi_audio"
89
-
83
if (
84
model_config.tokenizer_mode == "auto"
85
and model_config.model_impl == "terratorch"
vllm/tokenizers/registry.py
-12
@@ -159,18 +159,6 @@ def resolve_tokenizer_args(
159
):
160
tokenizer_mode = "mistral"
161
162
- # Try to use Grok2 tiktoken tokenizer if possible
163
- if tokenizer_mode == "auto" and any_pattern_in_repo_files(
164
- model_name_or_path=str(tokenizer_name),
165
- allow_patterns=["tokenizer.tok.json"],
166
- revision=revision,
167
- ):
168
- tokenizer_mode = "grok2"
169
-
170
- # Model-specific tokenizers
171
- if tokenizer_mode == "auto" and "/Qwen-VL" in str(tokenizer_name):
172
- tokenizer_mode = "qwen_vl"
173
-
162
# Fallback to HF tokenizer
163
if tokenizer_mode == "auto":
164
tokenizer_mode = "hf"
vllm/transformers_utils/processors/glm4v.py
+3
@@ -1,5 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://github.com/zai-org/CogAgent
6
from transformers import PreTrainedTokenizer
7
from transformers.image_processing_utils_fast import BaseImageProcessorFast
8
from transformers.image_utils import PILImageResampling
vllm/transformers_utils/processors/kimi_audio.py
+20
-78
@@ -1,10 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-# ruff: noqa
5
-# mypy: ignore-errors
6
-# coding=utf-8
7
-# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team. All rights reserved.
4
+# Copyright 2026 The Moonshot AI team and the HuggingFace Inc. team.
5
+# All rights reserved.
6
#
7
# Licensed under the Apache License, Version 2.0 (the "License");
8
# you may not use this file except in compliance with the License.
@@ -19,42 +17,13 @@
17
# limitations under the License.
18
"""Processor for Kimi-Audio ASR model."""
19
22
-from collections.abc import Mapping
23
-from typing import Any
24
-
20
import numpy as np
26
-import torch
27
-from transformers import AutoFeatureExtractor, BatchFeature, ProcessorMixin
21
+from transformers import BatchFeature, ProcessorMixin
22
from transformers.audio_utils import AudioInput
29
-from transformers.tokenization_utils_base import TextInput
30
-
31
-from vllm.tokenizers.kimi_audio import KimiAudioTokenizer
32
-
33
-
34
-def _get_feat_extract_output_lengths(input_lengths: torch.Tensor) -> torch.Tensor:
35
- """Compute output lengths after Whisper feature extraction."""
36
- input_lengths_leave = input_lengths % 100
37
- feat_lengths = (input_lengths_leave - 1) // 2 + 1
38
- output_lengths = (
39
- ((feat_lengths - 1) // 2 + 1 - 1) // 2 + 1 + (input_lengths // 100) * 13
40
- )
41
- return output_lengths
23
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
24
25
26
class KimiAudioProcessor(ProcessorMixin):
45
- r"""
46
- Constructs a Kimi-Audio processor.
47
-
48
- [`KimiAudioProcessor`] offers all the functionalities of [`WhisperFeatureExtractor`], and a tokenizer.
49
- See the [`~KimiAudioProcessor.__call__`] and [`~KimiAudioProcessor.decode`] for more information.
50
-
51
- Args:
52
- feature_extractor ([`WhisperFeatureExtractor`], *optional*):
53
- The audio feature extractor.
54
- tokenizer ([`PreTrainedTokenizer`], *optional*):
55
- The text tokenizer.
56
- """
57
-
27
# Required for ProcessorMixin
28
attributes = ["feature_extractor", "tokenizer"]
29
feature_extractor_class = "AutoFeatureExtractor"
@@ -69,44 +38,30 @@ class KimiAudioProcessor(ProcessorMixin):
38
AUDIO_SEQ_LEN: int = 376
39
40
def __init__(self, feature_extractor=None, tokenizer=None, **kwargs):
72
- # Pass feature_extractor and tokenizer to parent ProcessorMixin
73
- super().__init__(
74
- feature_extractor=feature_extractor,
75
- tokenizer=tokenizer,
76
- **kwargs,
77
- )
78
-
79
- def check_argument_for_proper_class(self, attribute_name: str, argument: Any):
80
- """Override to skip class validation for custom tokenizer."""
81
- # Skip validation for tokenizer since KimiAudioTokenizer doesn't inherit
82
- # from PreTrainedTokenizerBase but is compatible
83
- if attribute_name == "tokenizer" and argument is not None:
84
- return
85
- # For other attributes, use default validation
86
- super().check_argument_for_proper_class(attribute_name, argument)
41
+ self.feature_extractor = feature_extractor
42
+ self.tokenizer = tokenizer
43
44
def __call__(
45
self,
90
- text: TextInput = None,
91
- audio: AudioInput = None,
46
+ text: TextInput
47
+ | PreTokenizedInput
48
+ | list[TextInput]
49
+ | list[PreTokenizedInput]
50
+ | None = None,
51
+ audio: AudioInput | None = None,
52
return_tensors: str = "pt",
53
**kwargs,
54
) -> BatchFeature:
95
- """
96
- Main method to prepare for the model one or several sequences(s) and audio(s).
55
+ if text is not None:
56
+ if not isinstance(text, list):
57
+ text = [text]
58
98
- Args:
99
- text (`str`, `List[str]`):
100
- The sequence or batch of sequences to be encoded.
101
- audio (`np.ndarray`, `List[np.ndarray]`):
102
- The audio or batch of audio to be prepared. Each audio can be a NumPy array.
103
- return_tensors (`str`):
104
- The type of tensors to return ("pt", "np", etc.)
105
- """
106
- if text is None:
107
- raise ValueError("You need to specify either a `text` input to process.")
59
+ text_inputs = self.tokenizer(
60
+ text, return_tensors=return_tensors, padding=True
61
+ )
62
+ else:
63
+ text_inputs = {}
64
109
- # Process audio if provided
65
if audio is not None:
66
# Ensure audio is a list
67
if isinstance(audio, np.ndarray):
@@ -144,19 +99,6 @@ class KimiAudioProcessor(ProcessorMixin):
99
else:
100
audio_inputs = {}
101
147
- # Handle text input - can be string or token IDs from vLLM processor
148
- if isinstance(text, list) and len(text) > 0 and isinstance(text[0], int):
149
- # Text is already token IDs (from vLLM processor) - just wrap
150
- text_inputs = {"input_ids": torch.tensor([text], dtype=torch.long)}
151
- else:
152
- # Text is string - tokenize
153
- if not isinstance(text, list):
154
- text = [text]
155
-
156
- text_inputs = self.tokenizer(
157
- text, return_tensors=return_tensors, padding=True
158
- )
159
-
102
return BatchFeature(
103
data={**text_inputs, **audio_inputs},
104
tensor_type=return_tensors,
vllm/transformers_utils/processors/qwen_vl.py
+4
@@ -1,5 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Adapted from
5
+# https://huggingface.co/Qwen/Qwen-VL/blob/main/modeling_qwen.py
6
+# Copyright (c) Alibaba Cloud.
7
from transformers.image_processing_utils_fast import BaseImageProcessorFast
8
from transformers.image_utils import PILImageResampling
9
from transformers.processing_utils import ProcessorMixin