A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 01:30 UTC
[Misc] Move processors to `transformers_utils` (#35953)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Mar 9, 2026 at 11:31 UTC
d62856b9283b5f5a90e6f135b787e63b5ca3f157
13 files changed
+509
-597
vllm/model_executor/models/glm4v.py
+8
-73
@@ -13,11 +13,7 @@ import numpy as np
13
import torch
14
from torch import nn
15
from torch.nn import LayerNorm
16
-from torchvision import transforms
17
-from torchvision.transforms import InterpolationMode
18
-from transformers import BatchFeature, PreTrainedTokenizer, TensorType
19
-from transformers.image_utils import ImageInput
20
-from transformers.tokenization_utils_base import TextInput
16
+from transformers import BatchFeature
17
18
from vllm.config import VllmConfig
19
from vllm.config.multimodal import BaseDummyOptions
@@ -50,7 +46,8 @@ from vllm.multimodal.processing import (
46
PromptUpdate,
47
)
48
from vllm.sequence import IntermediateTensors
53
-from vllm.transformers_utils.configs import ChatGLMConfig
49
+from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
50
+from vllm.transformers_utils.processors.glm4v import GLM4VProcessor
51
from vllm.utils.tensor_schema import TensorSchema, TensorShape
52
53
from .chatglm import ChatGLMBaseModel, ChatGLMModel, GLMTransformer
@@ -386,81 +383,19 @@ class GLM4VModel(ChatGLMModel):
383
)
384
385
389
-class GLM4VProcessor:
390
- """
391
- This model doesn't define its own HF processor,
392
- so we implement our own one here.
393
- """
394
-
395
- def __init__(
396
- self,
397
- config: ChatGLMConfig,
398
- tokenizer: PreTrainedTokenizer,
399
- ) -> None:
400
- super().__init__()
401
-
402
- self.config = config
403
- self.tokenizer = tokenizer
404
-
405
- vision_config = config.vision_config
406
- image_size = vision_config["image_size"]
407
-
408
- self.image_transform = transforms.Compose(
409
- [
410
- transforms.Resize(
411
- (image_size, image_size),
412
- interpolation=InterpolationMode.BICUBIC,
413
- ),
414
- transforms.ToTensor(),
415
- transforms.Normalize(
416
- mean=(0.48145466, 0.4578275, 0.40821073),
417
- std=(0.26862954, 0.26130258, 0.27577711),
418
- ),
419
- ]
420
- )
421
-
422
- def __call__(
423
- self,
424
- text: TextInput | list[TextInput] | None = None,
425
- images: ImageInput | list[ImageInput] | None = None,
426
- return_tensors: str | TensorType | None = None,
427
- ) -> BatchFeature:
428
- if text is None:
429
- text = []
430
- if not isinstance(text, list):
431
- text = [text]
432
- if images is None:
433
- images = []
434
- if not isinstance(images, list):
435
- images = [images]
436
-
437
- text_inputs = self.tokenizer(text)
438
-
439
- if len(images) == 0:
440
- image_inputs = {}
441
- else:
442
- pixel_values = [self.image_transform(image) for image in images]
443
- image_inputs = {"pixel_values": torch.stack(pixel_values)}
444
-
445
- return BatchFeature(
446
- {
447
- **text_inputs,
448
- **image_inputs,
449
- },
450
- tensor_type=return_tensors,
451
- )
452
-
453
-
386
class GLM4VProcessingInfo(BaseProcessingInfo):
387
def get_hf_config(self):
388
return self.ctx.get_hf_config(ChatGLMConfig)
389
390
def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
391
+ config = self.get_hf_config()
392
+ vision_config = config.vision_config
393
+ image_size = vision_config["image_size"]
394
+
395
return self.ctx.init_processor(
396
GLM4VProcessor,
461
- config=self.get_hf_config(),
397
tokenizer=self.get_tokenizer(),
463
- **kwargs,
398
+ **{**kwargs, "image_size": image_size},
399
)
400
401
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/model_executor/models/molmo.py
+100
-180
@@ -4,7 +4,7 @@
4
import math
5
from collections.abc import Iterable, Mapping, Sequence
6
from dataclasses import dataclass
7
-from functools import cached_property, partial
7
+from functools import partial
8
from itertools import islice
9
from typing import Annotated
10
@@ -13,9 +13,11 @@ import torch
13
import torch.nn as nn
14
import torch.nn.functional as F
15
from einops import rearrange
16
-from transformers import BatchFeature, PretrainedConfig, ProcessorMixin, TensorType
17
-from transformers.image_utils import ImageInput
18
-from transformers.tokenization_utils_base import TextInput
16
+from transformers import (
17
+ BaseImageProcessor,
18
+ BatchFeature,
19
+ PretrainedConfig,
20
+)
21
22
from vllm.compilation.decorators import support_torch_compile
23
from vllm.config import CacheConfig, VllmConfig
@@ -1017,117 +1019,28 @@ def select_tiling(
1019
return candidate_tilings[ix]
1020
1021
1020
-class MolmoProcessorWrapper:
1021
- """
1022
- Wraps `MolmoProcessor` so that it can be called directly.
1023
-
1024
- The original definition can be found here:
1025
- https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/preprocessing_molmo.py
1026
- """
1027
-
1028
- def __init__(self, processor: ProcessorMixin):
1029
- super().__init__()
1030
-
1031
- self.processor = processor
1032
-
1033
- @cached_property
1034
- def vocab(self) -> dict[str, int]:
1035
- return self.processor.tokenizer.vocab # type: ignore
1036
-
1037
- @cached_property
1038
- def max_crops(self) -> int:
1039
- image_processor = self.processor.image_processor # type: ignore
1040
-
1041
- max_crops = image_processor.max_crops
1042
- assert isinstance(max_crops, int)
1043
-
1044
- return max_crops
1045
-
1046
- @cached_property
1047
- def base_image_input_size(self) -> tuple[int, int]:
1048
- image_processor = self.processor.image_processor # type: ignore
1049
-
1050
- base_image_input_size = image_processor.base_image_input_size
1051
- if isinstance(base_image_input_size, int):
1052
- return base_image_input_size, base_image_input_size
1053
-
1054
- return tuple(base_image_input_size)
1055
-
1056
- @cached_property
1057
- def image_patch_size(self) -> int:
1058
- image_processor = self.processor.image_processor # type: ignore
1059
-
1060
- image_patch_size = image_processor.image_patch_size
1061
- assert isinstance(image_patch_size, int)
1062
-
1063
- return image_patch_size
1064
-
1065
- @cached_property
1066
- def overlap_margins(self) -> tuple[int, int]:
1067
- image_processor = self.processor.image_processor # type: ignore
1068
-
1069
- left_margin, right_margin = image_processor.overlap_margins
1070
- assert isinstance(left_margin, int)
1071
- assert isinstance(right_margin, int)
1072
-
1073
- return left_margin, right_margin
1074
-
1075
- @cached_property
1076
- def image_token_length_w(self) -> int:
1077
- image_processor = self.processor.image_processor # type: ignore
1022
+def _as_2tuple(x: int | tuple[int, int]) -> tuple[int, int]:
1023
+ if isinstance(x, int):
1024
+ return x, x
1025
1079
- image_token_length_w = image_processor.image_token_length_w
1080
- assert isinstance(image_token_length_w, int)
1081
-
1082
- return image_token_length_w
1083
-
1084
- @cached_property
1085
- def image_token_length_h(self) -> int:
1086
- image_processor = self.processor.image_processor # type: ignore
1087
-
1088
- image_token_length_h = image_processor.image_token_length_h
1089
- assert isinstance(image_token_length_h, int)
1090
-
1091
- return image_token_length_h
1092
-
1093
- @property
1094
- def message_format(self) -> str | None:
1095
- return "role"
1096
-
1097
- @property
1098
- def always_start_with_space(self) -> bool:
1099
- return True
1100
-
1101
- @cached_property
1102
- def image_patch_id(self) -> int:
1103
- return self.vocab[IMAGE_PATCH_TOKEN]
1104
-
1105
- @cached_property
1106
- def im_col_id(self) -> int:
1107
- return self.vocab[IM_COL_TOKEN]
1026
+ return x
1027
1109
- @cached_property
1110
- def im_start_id(self) -> int:
1111
- return self.vocab[IM_START_TOKEN]
1028
1113
- @cached_property
1114
- def im_end_id(self) -> int:
1115
- return self.vocab[IM_END_TOKEN]
1116
-
1117
- @property
1118
- def pooling_size(self) -> int:
1119
- return POOLING_SIZE
1029
+class MolmoProcessingInfo(BaseProcessingInfo):
1030
+ def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1031
+ return {"image": None}
1032
1033
def select_tiling(
1034
self,
1035
*,
1036
image_width: int,
1037
image_height: int,
1038
+ image_processor: BaseImageProcessor,
1039
) -> tuple[int, int]:
1127
- max_crops = self.max_crops
1128
- left_margin, right_margin = self.overlap_margins
1129
- base_image_input_size = self.base_image_input_size
1130
- base_image_input_d = self.image_patch_size
1040
+ max_crops = image_processor.max_crops
1041
+ left_margin, right_margin = image_processor.overlap_margins
1042
+ base_image_input_size = _as_2tuple(image_processor.base_image_input_size)
1043
+ base_image_input_d = image_processor.image_patch_size
1044
1045
total_margin_pixels = base_image_input_d * (right_margin + left_margin)
1046
crop_patches = base_image_input_size[0] // base_image_input_d
@@ -1147,16 +1060,18 @@ class MolmoProcessorWrapper:
1060
*,
1061
image_width: int,
1062
image_height: int,
1063
+ image_processor: BaseImageProcessor,
1064
) -> tuple[int, int]:
1151
- left_margin, right_margin = self.overlap_margins
1152
- base_image_input_size = self.base_image_input_size
1153
- base_image_input_d = self.image_patch_size
1154
- pooling_size = self.pooling_size
1065
+ left_margin, right_margin = image_processor.overlap_margins
1066
+ base_image_input_size = _as_2tuple(image_processor.base_image_input_size)
1067
+ base_image_input_d = image_processor.image_patch_size
1068
+ pooling_size = POOLING_SIZE
1069
1070
crop_patches = base_image_input_size[0] // base_image_input_d
1071
tiling_w, tiling_h = self.select_tiling(
1072
image_height=image_height,
1073
image_width=image_width,
1074
+ image_processor=image_processor,
1075
)
1076
1077
nrows, ncols = get_patches_grid_size(
@@ -1170,70 +1085,22 @@ class MolmoProcessorWrapper:
1085
1086
return ncols, nrows
1087
1173
- def __call__(
1174
- self,
1175
- text: TextInput | list[TextInput] | None = None,
1176
- images: ImageInput | list[ImageInput] | None = None,
1177
- return_tensors: str | TensorType | None = None,
1178
- **kwargs,
1179
- ) -> BatchFeature:
1180
- outputs = self.processor.process( # type: ignore
1181
- text, images, **kwargs
1182
- )
1183
-
1184
- if images is None:
1185
- images = []
1186
- if not isinstance(images, list):
1187
- images = [images]
1188
-
1189
- input_ids: torch.Tensor = outputs.pop("input_ids")
1190
- outputs["input_ids"] = input_ids.unsqueeze(0)
1191
-
1192
- image_input_idx = outputs.pop("image_input_idx", None)
1193
- if image_input_idx is not None:
1194
- feat_is_patch = image_input_idx >= 0
1195
-
1196
- tilings = [
1197
- self.select_tiling(
1198
- image_width=image.size[0],
1199
- image_height=image.size[1],
1200
- )
1201
- for image in images
1202
- ]
1203
- # For each image: tiling_h * tiling_w + extra
1204
- num_crops = torch.tensor(tilings).prod(-1) + 1
1205
- assert num_crops.sum() == len(feat_is_patch)
1206
-
1207
- outputs["image_input_idx"] = image_input_idx
1208
- outputs["num_crops"] = num_crops
1209
- outputs["img_patch_id"] = self.image_patch_id
1210
-
1211
- return BatchFeature(outputs)
1212
-
1213
-
1214
-class MolmoProcessingInfo(BaseProcessingInfo):
1215
- def get_hf_processor(self, **kwargs: object) -> MolmoProcessorWrapper:
1216
- processor = self.ctx.get_hf_processor(**kwargs)
1217
- return MolmoProcessorWrapper(processor)
1218
-
1219
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
1220
- return {"image": None}
1221
-
1088
def get_num_image_tokens(
1089
self,
1090
*,
1091
image_width: int,
1092
image_height: int,
1227
- processor: MolmoProcessorWrapper,
1093
+ image_processor: BaseImageProcessor,
1094
) -> int:
1229
- ncols, nrows = processor.get_patches_grid_size(
1095
+ ncols, nrows = self.get_patches_grid_size(
1096
image_width=image_width,
1097
image_height=image_height,
1098
+ image_processor=image_processor,
1099
)
1233
- pooling_size = processor.pooling_size
1100
+ pooling_size = POOLING_SIZE
1101
1235
- image_token_length_w = processor.image_token_length_w
1236
- image_token_length_h = processor.image_token_length_h
1102
+ image_token_length_w = image_processor.image_token_length_w
1103
+ image_token_length_h = image_processor.image_token_length_h
1104
1105
# Calculate total tokens: 2 for start/end + (w+1)*h for column separators
1106
extra = 2 + (image_token_length_w + 1) * image_token_length_h
@@ -1243,9 +1110,10 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1110
1111
def get_image_size_with_most_features(self) -> ImageSize:
1112
processor = self.get_hf_processor()
1113
+ image_processor = processor.image_processor
1114
1247
- tilings = get_candidate_tilings(processor.max_crops)
1248
- base_h, base_w = processor.base_image_input_size
1115
+ tilings = get_candidate_tilings(image_processor.max_crops)
1116
+ base_h, base_w = _as_2tuple(image_processor.base_image_input_size)
1117
1118
largest_feature_size, largest_feature_pinpoint = 0, None
1119
for wr, hr in tilings:
@@ -1254,7 +1122,7 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1122
feat_size = self.get_num_image_tokens(
1123
image_width=width,
1124
image_height=height,
1257
- processor=processor,
1125
+ image_processor=image_processor,
1126
)
1127
if feat_size > largest_feature_size:
1128
largest_feature_size = feat_size
@@ -1292,6 +1160,54 @@ class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1160
1161
1162
class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1163
+ def _call_hf_processor(
1164
+ self,
1165
+ prompt: str,
1166
+ mm_data: Mapping[str, object],
1167
+ mm_kwargs: Mapping[str, object],
1168
+ tok_kwargs: Mapping[str, object],
1169
+ ) -> BatchFeature:
1170
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
1171
+ processed_outputs = self.info.ctx.call_hf_processor(
1172
+ hf_processor.process,
1173
+ dict(text=prompt, **mm_data),
1174
+ dict(**mm_kwargs, **tok_kwargs),
1175
+ )
1176
+
1177
+ tokenizer = hf_processor.tokenizer
1178
+ image_patch_id = tokenizer.vocab[IMAGE_PATCH_TOKEN]
1179
+
1180
+ image_processor = hf_processor.image_processor
1181
+
1182
+ input_ids: torch.Tensor = processed_outputs.pop("input_ids")
1183
+ processed_outputs["input_ids"] = input_ids.unsqueeze(0)
1184
+
1185
+ if (images := mm_data.get("images")) is not None:
1186
+ mm_items = self.info.parse_mm_data({"image": images}, validate=False)
1187
+ parsed_images = mm_items.get_items("image", ImageProcessorItems)
1188
+ image_sizes = [
1189
+ parsed_images.get_image_size(i) for i in range(len(parsed_images))
1190
+ ]
1191
+
1192
+ feat_is_patch = processed_outputs["image_input_idx"] >= 0
1193
+
1194
+ tilings = [
1195
+ self.info.select_tiling(
1196
+ image_width=image_size.width,
1197
+ image_height=image_size.height,
1198
+ image_processor=image_processor,
1199
+ )
1200
+ for image_size in image_sizes
1201
+ ]
1202
+ # For each image: tiling_h * tiling_w + extra
1203
+ num_crops = torch.tensor(tilings).prod(-1) + 1
1204
+ assert num_crops.sum() == len(feat_is_patch)
1205
+
1206
+ processed_outputs["num_crops"] = num_crops
1207
+ processed_outputs["img_patch_id"] = image_patch_id
1208
+
1209
+ return processed_outputs
1210
+
1211
def _apply_hf_processor_tokens_only(
1212
self,
1213
prompt_tokens: list[int],
@@ -1301,18 +1217,19 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1217
# The chat template is already applied to the prompt tokens
1218
# Use message_format="none" to avoid applying it again
1219
# Prepend an empty space if `always_start_with_space` is True
1304
- tokens = processor.processor.get_tokens_input( # type: ignore
1220
+ tokens = processor.get_tokens_input(
1221
self.info.get_tokenizer().decode(prompt_tokens),
1222
message_format="none",
1307
- always_start_with_space=processor.always_start_with_space,
1223
+ always_start_with_space=True,
1224
)
1225
1226
# Prepend a BOS token id to the tokens
1227
processed_data = self.info.ctx.call_hf_processor(
1312
- processor, # type: ignore
1228
+ processor.process,
1229
dict(tokens=tokens),
1230
)
1315
- (prompt_ids,) = processed_data.pop("input_ids").tolist()
1231
+ prompt_ids = processed_data.pop("input_ids").tolist()
1232
+ print(prompt_ids, len(prompt_ids))
1233
1234
return prompt_ids
1235
@@ -1338,16 +1255,18 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1255
hf_processor_mm_kwargs: Mapping[str, object],
1256
out_mm_kwargs: MultiModalKwargsItems,
1257
) -> Sequence[PromptUpdate]:
1341
- processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1342
-
1343
- image_token_length_w = processor.image_token_length_w
1344
- image_token_length_h = processor.image_token_length_h
1345
- pooling_size = processor.pooling_size
1258
+ tokenizer = self.info.get_tokenizer()
1259
+ vocab = tokenizer.get_vocab()
1260
+ img_patch_id = vocab[IMAGE_PATCH_TOKEN]
1261
+ img_col_id = vocab[IM_COL_TOKEN]
1262
+ img_start_id = vocab[IM_START_TOKEN]
1263
+ img_end_id = vocab[IM_END_TOKEN]
1264
1347
- img_patch_id = processor.image_patch_id
1348
- img_col_id = processor.im_col_id
1349
- img_start_id = processor.im_start_id
1350
- img_end_id = processor.im_end_id
1265
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1266
+ image_processor = processor.image_processor
1267
+ image_token_length_w = image_processor.image_token_length_w
1268
+ image_token_length_h = image_processor.image_token_length_h
1269
+ pooling_size = POOLING_SIZE
1270
1271
extra_row = [img_patch_id] * image_token_length_w + [img_col_id]
1272
extra_joint = [img_start_id] + extra_row * image_token_length_h + [img_end_id]
@@ -1356,9 +1275,10 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1275
images = mm_items.get_items("image", ImageProcessorItems)
1276
image_size = images.get_image_size(item_idx)
1277
1359
- ncols, nrows = processor.get_patches_grid_size(
1278
+ ncols, nrows = self.info.get_patches_grid_size(
1279
image_width=image_size.width,
1280
image_height=image_size.height,
1281
+ image_processor=image_processor,
1282
)
1283
1284
joint_row = [img_patch_id] * ((ncols + 1) // pooling_size) + [img_col_id]
vllm/model_executor/models/pixtral.py
+12
-111
@@ -4,7 +4,6 @@
4
import math
5
from collections.abc import Iterable, Mapping, Sequence
6
from dataclasses import dataclass, fields
7
-from functools import cached_property
7
from typing import Annotated, Literal
8
9
import torch
@@ -13,10 +12,7 @@ import torch.nn.functional as F
12
from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk
13
from mistral_common.protocol.instruct.messages import UserMessage
14
from mistral_common.protocol.instruct.request import ChatCompletionRequest
16
-from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
17
-from PIL import Image
18
-from transformers import BatchFeature, PixtralVisionConfig, TensorType
19
-from transformers.image_utils import ImageInput
15
+from transformers import PixtralVisionConfig
16
from transformers.models.pixtral.image_processing_pixtral import (
17
_num_image_tokens as _get_pixtral_hf_num_image_tokens,
18
)
@@ -25,7 +21,6 @@ from transformers.models.pixtral.modeling_pixtral import (
21
apply_rotary_pos_emb,
22
position_ids_in_meshgrid,
23
)
28
-from transformers.tokenization_utils_base import TextInput
24
25
from vllm.config import VllmConfig
26
from vllm.config.multimodal import BaseDummyOptions
@@ -66,6 +61,7 @@ from vllm.platforms import current_platform
61
from vllm.sequence import IntermediateTensors
62
from vllm.tokenizers import cached_tokenizer_from_config
63
from vllm.tokenizers.mistral import MistralTokenizer
64
+from vllm.transformers_utils.processors.pixtral import MistralCommonPixtralProcessor
65
from vllm.utils.tensor_schema import TensorSchema, TensorShape
66
67
from .interfaces import (
@@ -121,93 +117,6 @@ class PixtralImagePixelInputs(TensorSchema):
117
]
118
119
124
-class PixtralProcessorAdapter:
125
- """
126
- Provide a HF-compatible interface for
127
- `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
128
- """
129
-
130
- def __init__(self, tokenizer: MistralTokenizer) -> None:
131
- super().__init__()
132
-
133
- self.tokenizer = tokenizer
134
-
135
- @property
136
- def image_processor(self) -> ImageEncoder:
137
- image_encoder = self.tokenizer.instruct.mm_encoder
138
- assert isinstance(image_encoder, ImageEncoder)
139
- return image_encoder
140
-
141
- @cached_property
142
- def image_break_id(self) -> int:
143
- return self.image_processor.special_ids.img_break
144
-
145
- @cached_property
146
- def image_token_id(self) -> int:
147
- return self.image_processor.special_ids.img
148
-
149
- @cached_property
150
- def image_end_id(self) -> int:
151
- return self.image_processor.special_ids.img_end
152
-
153
- @cached_property
154
- def image_size(self) -> int:
155
- return self.image_processor.mm_config.max_image_size
156
-
157
- @cached_property
158
- def patch_size(self) -> int:
159
- return self.image_processor.mm_config.image_patch_size
160
-
161
- def __call__(
162
- self,
163
- text: TextInput | list[TextInput] | None = None,
164
- images: ImageInput | list[ImageInput] | None = None,
165
- return_tensors: str | TensorType | None = None,
166
- **kwargs,
167
- ) -> Mapping[str, NestedTensors]:
168
- if text is None:
169
- text = []
170
- if not isinstance(text, list):
171
- text = [text]
172
- if images is None:
173
- images = []
174
- if not isinstance(images, list):
175
- images = [images]
176
-
177
- if not images:
178
- input_ids = self.tokenizer(text).input_ids
179
-
180
- return {"input_ids": torch.tensor(input_ids)}
181
-
182
- # Allow dummy text, which is used for profiling as well as token inputs
183
- if any(len(t) > 0 for t in text):
184
- raise ValueError(
185
- "You've passed text inputs instead of token inputs. "
186
- "Make sure to process your input via `mistral_common`'s "
187
- "tokenizer or pass a chat completion request. "
188
- "For more info, see: "
189
- "https://github.com/vllm-project/vllm/issues/8411."
190
- )
191
-
192
- images_processed = list[torch.Tensor]()
193
- images_tokens = list[torch.Tensor]()
194
-
195
- for image in images:
196
- image_inputs = self.image_processor(ImageChunk(image=image))
197
- image_processed = torch.tensor(image_inputs.image)
198
- image_tokens = torch.tensor(image_inputs.tokens)
199
-
200
- images_processed.append(image_processed)
201
- images_tokens.append(image_tokens)
202
-
203
- return BatchFeature(
204
- {
205
- "input_ids": torch.cat(images_tokens)[None].expand(len(text), -1),
206
- "images": images_processed,
207
- }
208
- )
209
-
210
-
120
class PixtralProcessingInfo(BaseProcessingInfo):
121
def get_tokenizer(self) -> MistralTokenizer:
122
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
@@ -216,28 +125,19 @@ class PixtralProcessingInfo(BaseProcessingInfo):
125
126
return tokenizer
127
219
- def get_hf_processor(self) -> PixtralProcessorAdapter:
220
- return PixtralProcessorAdapter(self.get_tokenizer())
128
+ def get_hf_processor(self, **kwargs) -> MistralCommonPixtralProcessor:
129
+ return self.ctx.init_processor(
130
+ MistralCommonPixtralProcessor,
131
+ tokenizer=self.get_tokenizer(),
132
+ **kwargs,
133
+ )
134
135
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
136
return {"image": None}
137
225
- def get_num_image_tokens(
226
- self,
227
- *,
228
- image_width: int,
229
- image_height: int,
230
- processor: PixtralProcessorAdapter,
231
- ) -> int:
232
- ncols, nrows = processor.image_processor._image_to_num_tokens(
233
- Image.new("RGB", (image_width, image_height))
234
- )
235
-
236
- return ncols * nrows
237
-
138
def get_image_size_with_most_features(self) -> ImageSize:
139
image_processor = self.get_hf_processor().image_processor
240
- max_image_size = image_processor.mm_config.max_image_size
140
+ max_image_size = image_processor.mm_encoder.mm_config.max_image_size
141
142
return ImageSize(width=max_image_size, height=max_image_size)
143
@@ -321,8 +221,9 @@ class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo])
221
images = mm_items.get_items("image", ImageProcessorItems)
222
image_size = images.get_image_size(item_idx)
223
324
- ncols, nrows = processor.image_processor._image_to_num_tokens(
325
- Image.new("RGB", (image_size.width, image_size.height))
224
+ _, nrows, ncols = processor.image_processor.get_number_of_image_patches(
225
+ image_size.height,
226
+ image_size.width,
227
)
228
229
tokens = ([image_token_id] * ncols + [image_break_id]) * nrows
vllm/model_executor/models/qwen_vl.py
+6
-89
@@ -14,11 +14,7 @@ from typing import Annotated, Literal, TypeAlias
14
import regex as re
15
import torch
16
from torch import nn
17
-from torchvision import transforms
18
-from torchvision.transforms import InterpolationMode
19
-from transformers import BatchFeature, PretrainedConfig, PreTrainedTokenizer, TensorType
20
-from transformers.image_utils import ImageInput
21
-from transformers.tokenization_utils_base import TextInput
17
+from transformers import BatchFeature
18
19
from vllm.config import VllmConfig
20
from vllm.config.multimodal import BaseDummyOptions
@@ -48,6 +44,7 @@ from vllm.multimodal.processing import (
44
PromptUpdateDetails,
45
)
46
from vllm.sequence import IntermediateTensors
47
+from vllm.transformers_utils.processors.qwen_vl import QwenVLProcessor
48
from vllm.utils.tensor_schema import TensorSchema, TensorShape
49
50
from .interfaces import (
@@ -434,96 +431,16 @@ class QwenVLModel(QWenModel):
431
)
432
433
437
-class QwenVLProcessor:
438
- """
439
- This model doesn't define its own HF processor,
440
- so we implement our own one here.
441
-
442
- We call the wrapped tokenizer to automatically insert image pad tokens:
443
- https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
444
-
445
- The image processor is defined here:
446
- https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
447
- """
448
-
449
- def __init__(
450
- self,
451
- config: PretrainedConfig,
452
- tokenizer: PreTrainedTokenizer,
453
- ) -> None:
454
- super().__init__()
455
-
456
- self.config = config
457
- self.tokenizer = tokenizer
458
-
434
+class QwenVLProcessingInfo(BaseProcessingInfo):
435
+ def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
436
+ config = self.get_hf_config()
437
vision_config = config.visual
438
image_size = vision_config["image_size"]
439
462
- self.image_transform = transforms.Compose(
463
- [
464
- transforms.Resize(
465
- (image_size, image_size),
466
- interpolation=InterpolationMode.BICUBIC,
467
- ),
468
- transforms.ToTensor(),
469
- transforms.Normalize(
470
- mean=(0.48145466, 0.4578275, 0.40821073),
471
- std=(0.26862954, 0.26130258, 0.27577711),
472
- ),
473
- ]
474
- )
475
-
476
- @property
477
- def image_start_tag(self) -> str:
478
- return self.tokenizer.image_start_tag # type: ignore
479
-
480
- @property
481
- def image_end_tag(self) -> str:
482
- return self.tokenizer.image_end_tag # type: ignore
483
-
484
- @property
485
- def image_pad_tag(self) -> str:
486
- return self.tokenizer.image_pad_tag # type: ignore
487
-
488
- def __call__(
489
- self,
490
- text: TextInput | list[TextInput] | None = None,
491
- images: ImageInput | list[ImageInput] | None = None,
492
- return_tensors: str | TensorType | None = None,
493
- ) -> BatchFeature:
494
- if text is None:
495
- text = []
496
- if not isinstance(text, list):
497
- text = [text]
498
- if images is None:
499
- images = []
500
- if not isinstance(images, list):
501
- images = [images]
502
-
503
- text_inputs = self.tokenizer(text)
504
-
505
- if len(images) == 0:
506
- image_inputs = {}
507
- else:
508
- pixel_values = [self.image_transform(image) for image in images]
509
- image_inputs = {"pixel_values": torch.stack(pixel_values)}
510
-
511
- return BatchFeature(
512
- {
513
- **text_inputs,
514
- **image_inputs,
515
- },
516
- tensor_type=return_tensors,
517
- )
518
-
519
-
520
-class QwenVLProcessingInfo(BaseProcessingInfo):
521
- def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
440
return self.ctx.init_processor(
441
QwenVLProcessor,
524
- config=self.get_hf_config(),
442
tokenizer=self.get_tokenizer(),
526
- **kwargs,
443
+ **{**kwargs, "image_size": image_size},
444
)
445
446
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
vllm/model_executor/models/voxtral.py
+40
-135
@@ -3,25 +3,19 @@
3
4
import math
5
from collections.abc import Iterable, Mapping, Sequence
6
-from functools import cached_property, partial
7
-from math import ceil
6
+from functools import partial
7
from typing import Literal, cast
8
9
import numpy as np
10
import regex as re
11
import torch
12
import torch.nn as nn
14
-from mistral_common.audio import mel_filter_bank
13
+from mistral_common.audio import Audio, mel_filter_bank
14
from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
15
from mistral_common.protocol.instruct.messages import UserMessage
16
from mistral_common.protocol.instruct.request import ChatCompletionRequest
17
from mistral_common.protocol.transcription.request import TranscriptionRequest
19
-from mistral_common.tokens.tokenizers.audio import (
20
- Audio,
21
- AudioEncoder,
22
-)
23
-from transformers import BatchFeature, TensorType, WhisperConfig
24
-from transformers.tokenization_utils_base import TextInput
18
+from transformers import BatchFeature, WhisperConfig
19
20
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
21
from vllm.config.multimodal import BaseDummyOptions
@@ -62,6 +56,7 @@ from vllm.multimodal.processing.processor import (
56
from vllm.sequence import IntermediateTensors
57
from vllm.tokenizers import cached_tokenizer_from_config
58
from vllm.tokenizers.mistral import MistralTokenizer
59
+from vllm.transformers_utils.processors.voxtral import MistralCommonVoxtralProcessor
60
61
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription
62
from .utils import init_vllm_registered_model, maybe_prefix
@@ -81,98 +76,6 @@ ISO639_1_SUPPORTED_LANGS = {
76
}
77
78
84
-class VoxtralProcessorAdapter:
85
- """
86
- Provide a HF-compatible interface for
87
- :class:`mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
88
- """
89
-
90
- def __init__(self, tokenizer: MistralTokenizer) -> None:
91
- super().__init__()
92
- self.tokenizer = tokenizer
93
-
94
- @cached_property
95
- def _audio_processor(self) -> AudioEncoder:
96
- audio_encoder = self.tokenizer.instruct.audio_encoder
97
- assert isinstance(audio_encoder, AudioEncoder)
98
- return audio_encoder
99
-
100
- @cached_property
101
- def audio_token_id(self) -> int:
102
- return self._audio_processor.special_ids.audio
103
-
104
- @cached_property
105
- def begin_audio_token_id(self) -> int:
106
- return self._audio_processor.special_ids.begin_audio
107
-
108
- @cached_property
109
- def sampling_rate(self) -> int:
110
- return self._audio_processor.audio_config.sampling_rate
111
-
112
- @cached_property
113
- def frame_rate(self) -> float:
114
- return self._audio_processor.audio_config.frame_rate
115
-
116
- def get_num_audio_tokens(
117
- self,
118
- audio_length: int,
119
- ) -> int:
120
- return ceil(audio_length / (self.sampling_rate // self.frame_rate))
121
-
122
- def __call__(
123
- self,
124
- text: TextInput | list[TextInput] | None = None,
125
- audios: np.ndarray | list[np.ndarray] | None = None,
126
- return_tensors: str | TensorType | None = None,
127
- **kwargs,
128
- ) -> Mapping[str, NestedTensors]:
129
- if text is None:
130
- text = []
131
- if not isinstance(text, list):
132
- text = [text]
133
- if audios is None:
134
- audios = []
135
- if not isinstance(audios, list):
136
- audios = [audios]
137
-
138
- if not audios:
139
- input_ids = self.tokenizer(text).input_ids
140
- return {"input_ids": torch.tensor(input_ids)}
141
-
142
- # Allow dummy text, which is used for profiling as well as token inputs
143
- if any(len(t) > 0 for t in text):
144
- raise ValueError(
145
- "You've passed text inputs instead of token inputs. "
146
- "Make sure to process your input via `mistral_common`'s "
147
- "tokenizer or pass a chat completion request. "
148
- "For more info, see: "
149
- "https://github.com/vllm-project/vllm/issues/8411."
150
- )
151
-
152
- audios_tokens = list[torch.Tensor]()
153
- audios_processed = list[torch.Tensor]()
154
- for audio in audios:
155
- assert isinstance(audio, np.ndarray)
156
- assert audio.ndim == 1
157
-
158
- if not self._audio_processor.audio_config.is_streaming:
159
- audio = self._audio_processor.pad(audio, self.sampling_rate)
160
-
161
- audio_tokens = [self.begin_audio_token_id] + [
162
- self.audio_token_id
163
- ] * self.get_num_audio_tokens(len(audio))
164
-
165
- audios_tokens.append(torch.tensor(audio_tokens))
166
- audios_processed.append(torch.tensor(audio))
167
-
168
- return BatchFeature(
169
- {
170
- "input_ids": torch.cat(audios_tokens)[None].expand(len(text), -1),
171
- "audio_arrays": audios_processed,
172
- }
173
- )
174
-
175
-
79
class VoxtralProcessingInfo(BaseProcessingInfo):
80
def get_tokenizer(self) -> MistralTokenizer:
81
tokenizer = cached_tokenizer_from_config(self.ctx.model_config)
@@ -181,12 +84,18 @@ class VoxtralProcessingInfo(BaseProcessingInfo):
84
85
return tokenizer
86
184
- def get_hf_processor(self) -> VoxtralProcessorAdapter:
185
- return VoxtralProcessorAdapter(self.get_tokenizer())
87
+ def get_hf_processor(self, **kwargs) -> MistralCommonVoxtralProcessor:
88
+ return self.ctx.init_processor(
89
+ MistralCommonVoxtralProcessor,
90
+ tokenizer=self.get_tokenizer(),
91
+ **kwargs,
92
+ )
93
94
def get_data_parser(self):
95
+ feature_extractor = self.get_hf_processor().feature_extractor
96
+
97
return MultiModalDataParser(
189
- target_sr=self.get_hf_processor().sampling_rate,
98
+ target_sr=feature_extractor.sampling_rate,
99
target_channels=1,
100
expected_hidden_size=self._get_expected_hidden_size(),
101
)
@@ -205,9 +114,10 @@ class VoxtralProcessingInfo(BaseProcessingInfo):
114
return self.ctx.model_config.max_model_len
115
116
def get_max_audio_array_len(self) -> int:
208
- processor = self.get_hf_processor()
117
+ feature_extractor = self.get_hf_processor().feature_extractor
118
+
119
return self.get_max_audio_tokens() * int(
210
- processor.sampling_rate // processor.frame_rate
120
+ feature_extractor.sampling_rate // feature_extractor.frame_rate
121
)
122
123
@@ -242,6 +152,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
152
mm_options: Mapping[str, BaseDummyOptions],
153
) -> ProcessorInputs:
154
tokenizer = self.info.get_tokenizer()
155
+ feature_extractor = self.info.get_hf_processor().feature_extractor
156
157
dummy_text = self.get_dummy_text(mm_counts)
158
dummy_mm_data = self.get_dummy_mm_data(seq_len, mm_counts, mm_options)
@@ -252,7 +163,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
163
for audio in dummy_audios:
164
audio_item = Audio(
165
audio_array=audio,
255
- sampling_rate=self.info.get_hf_processor().sampling_rate,
166
+ sampling_rate=feature_extractor.sampling_rate,
167
format=format,
168
)
169
chunk = AudioChunk(input_audio=RawAudio.from_audio(audio_item))
@@ -292,33 +203,26 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
203
# skip validation here
204
...
205
295
- def _apply_hf_processor_mm_only(
206
+ def _call_hf_processor(
207
self,
297
- mm_items: MultiModalDataItems,
298
- hf_processor_mm_kwargs: Mapping[str, object],
299
- tokenization_kwargs: Mapping[str, object],
208
+ prompt: str,
209
+ mm_data: Mapping[str, object],
210
+ mm_kwargs: Mapping[str, object],
211
+ tok_kwargs: Mapping[str, object],
212
) -> BatchFeature:
301
- processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
302
- processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
303
- audios = processor_data.get("audios", [])
304
- if not isinstance(audios, list):
305
- audios = [audios]
306
-
307
- audio_config = processor._audio_processor.audio_config
308
- audio_tensors: list[torch.Tensor] = []
309
- for audio in audios:
310
- audio = np.asarray(audio, dtype=np.float32).ravel()
311
- if not audio_config.is_streaming:
312
- audio = processor._audio_processor.pad(
313
- audio,
314
- processor.sampling_rate,
315
- audio_config.is_streaming,
316
- )
317
- audio_tensors.append(torch.tensor(audio))
318
-
319
- result = BatchFeature({"audio_arrays": audio_tensors} if audio_tensors else {})
320
- result.update(passthrough_data)
321
- return result
213
+ mm_data = dict(mm_data)
214
+ audios = mm_data.pop("audios", [])
215
+
216
+ if audios:
217
+ # MistralCommonVoxtralProcessor accepts "audio"
218
+ mm_data["audio"] = audios
219
+
220
+ return super()._call_hf_processor(
221
+ prompt=prompt,
222
+ mm_data=mm_data,
223
+ mm_kwargs=mm_kwargs,
224
+ tok_kwargs=tok_kwargs,
225
+ )
226
227
def _get_prompt_updates(
228
self,
@@ -327,6 +231,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
231
out_mm_kwargs: MultiModalKwargsItems,
232
) -> Sequence[PromptUpdate]:
233
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
234
+ feature_extractor = processor.feature_extractor
235
236
audio_id = processor.audio_token_id
237
out_mm_data = out_mm_kwargs.require_data()
@@ -348,7 +253,7 @@ class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo])
253
audios = mm_items.get_items("audio", AudioProcessorItems)
254
audio_len = audios.get_audio_length(item_idx)
255
351
- nb_audio_tokens = processor.get_num_audio_tokens(audio_len)
256
+ nb_audio_tokens = feature_extractor.get_num_audio_tokens(audio_len)
257
258
return [audio_id] * nb_audio_tokens
259
@@ -560,8 +465,8 @@ class VoxtralForConditionalGeneration(
465
This is used for estimating the amount of processing for this audio.
466
"""
467
tokenizer = cached_tokenizer_from_config(model_config)
563
- adapter = VoxtralProcessorAdapter(tokenizer)
564
- return adapter.get_num_audio_tokens(
468
+ adapter = MistralCommonVoxtralProcessor(tokenizer)
469
+ return adapter.feature_extractor.get_num_audio_tokens(
470
int(audio_duration_s * stt_config.sample_rate)
471
)
472
vllm/model_executor/models/voxtral_realtime.py
+2
-1
@@ -8,12 +8,13 @@ from typing import Literal
8
9
import numpy as np
10
import torch
11
+from mistral_common.audio import Audio
12
from mistral_common.protocol.instruct.chunk import RawAudio
13
from mistral_common.protocol.transcription.request import (
14
StreamingMode,
15
TranscriptionRequest,
16
)
16
-from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
17
+from mistral_common.tokens.tokenizers.audio import AudioConfig
18
19
from vllm.compilation.decorators import support_torch_compile
20
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
vllm/multimodal/processing/context.py
+5
-5
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
from abc import abstractmethod
5
-from collections.abc import Mapping
5
+from collections.abc import Callable, Mapping
6
from contextlib import contextmanager
7
from dataclasses import dataclass, field
8
from functools import cached_property
@@ -241,13 +241,13 @@ class InputProcessingContext:
241
242
def call_hf_processor(
243
self,
244
- hf_processor: ProcessorMixin,
244
+ hf_processor: Callable[..., BatchFeature] | ProcessorMixin,
245
data: Mapping[str, object],
246
kwargs: Mapping[str, object] = {},
247
*,
248
num_tries: int = 1,
249
max_tries: int = 5,
250
- ) -> BatchFeature | JSONTree:
250
+ ) -> BatchFeature:
251
"""
252
Call `hf_processor` on the prompt `data`
253
(text, image, audio...) with configurable options `kwargs`.
@@ -300,7 +300,7 @@ class InputProcessingContext:
300
301
if isinstance(output, BatchFeature):
302
output_ = self._postprocess_output(output.data)
303
- return BatchFeature(output_)
303
+ return BatchFeature(output_) # type: ignore
304
305
logger.warning_once(
306
"%s did not return `BatchFeature`. "
@@ -309,7 +309,7 @@ class InputProcessingContext:
309
type(hf_processor).__name__,
310
)
311
312
- return self._postprocess_output(output)
312
+ return self._postprocess_output(output) # type: ignore
313
314
315
class BaseProcessingInfo:
vllm/transformers_utils/processor.py
+10
-3
@@ -241,12 +241,13 @@ def get_processor_kwargs_type(
241
call_kwargs_annotations = call_kwargs.annotation if call_kwargs else None
242
243
# if the processor has explicit kwargs annotation, use it
244
- if call_kwargs_annotations not in (None, inspect._empty):
244
+ if call_kwargs_annotations not in (None, inspect._empty): # noqa: SIM102
245
# get_type_hints will parse all type annotations at runtime,
246
# and if an annotation refers to a type or
247
# name that hasn’t been imported or defined, it will raise an error.
248
# So we use __annotations__ to get the raw annotations directly.
249
- return get_args(call_kwargs_annotations)[0]
249
+ if anno_args := get_args(call_kwargs_annotations):
250
+ return anno_args[0]
251
252
# otherwise, try to get from ProcessorKwargs
253
module_name = type(processor).__module__
@@ -266,7 +267,13 @@ def get_processor_kwargs_keys(
267
kwargs_cls: type[processing_utils.ProcessingKwargs],
268
) -> set[str]:
269
dynamic_kwargs: set[str] = set()
269
- modality_kwargs = {"text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"}
270
+ modality_kwargs = {
271
+ "text_kwargs",
272
+ "images_kwargs",
273
+ "videos_kwargs",
274
+ "audio_kwargs",
275
+ "common_kwargs",
276
+ }
277
278
try:
279
# get kwargs annotations in processor
vllm/transformers_utils/processors/__init__.py
+8
@@ -15,10 +15,14 @@ _CLASS_TO_MODULE: dict[str, str] = {
15
"DeepseekVLV2Processor": "vllm.transformers_utils.processors.deepseek_vl2",
16
"FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2",
17
"FunASRProcessor": "vllm.transformers_utils.processors.funasr",
18
+ "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v",
19
"HunYuanVLProcessor": "vllm.transformers_utils.processors.hunyuan_vl",
20
"HunYuanVLImageProcessor": "vllm.transformers_utils.processors.hunyuan_vl_image",
21
+ "MistralCommonPixtralProcessor": "vllm.transformers_utils.processors.pixtral",
22
+ "MistralCommonVoxtralProcessor": "vllm.transformers_utils.processors.voxtral",
23
"OvisProcessor": "vllm.transformers_utils.processors.ovis",
24
"Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
25
+ "QwenVLProcessor": "vllm.transformers_utils.processors.qwen_vl",
26
"Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
27
}
28
@@ -28,10 +32,14 @@ __all__ = [
32
"DeepseekVLV2Processor",
33
"FireRedASR2Processor",
34
"FunASRProcessor",
35
+ "GLM4VProcessor",
36
"HunYuanVLProcessor",
37
"HunYuanVLImageProcessor",
38
+ "MistralCommonPixtralProcessor",
39
+ "MistralCommonVoxtralProcessor",
40
"OvisProcessor",
41
"Ovis2_5Processor",
42
+ "QwenVLProcessor",
43
"Qwen3ASRProcessor",
44
]
45
vllm/transformers_utils/processors/glm4v.py
new
+35
@@ -0,0 +1,35 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from transformers import PreTrainedTokenizer
4
+from transformers.image_processing_utils_fast import BaseImageProcessorFast
5
+from transformers.image_utils import PILImageResampling
6
+from transformers.processing_utils import ProcessorMixin
7
+
8
+
9
+class GLM4VImageProcessorFast(BaseImageProcessorFast):
10
+ """
11
+ Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177
12
+ to HF Transformers.
13
+ """
14
+
15
+ resample = PILImageResampling.BICUBIC
16
+ image_mean = [0.48145466, 0.4578275, 0.40821073]
17
+ image_std = [0.26862954, 0.26130258, 0.27577711]
18
+ size = {"height": 1120, "width": 1120}
19
+ do_resize = True
20
+ do_rescale = True
21
+ do_normalize = True
22
+
23
+
24
+class GLM4VProcessor(ProcessorMixin):
25
+ attributes = ["image_processor", "tokenizer"]
26
+
27
+ def __init__(
28
+ self,
29
+ tokenizer: PreTrainedTokenizer,
30
+ image_size: int,
31
+ ) -> None:
32
+ self.tokenizer = tokenizer
33
+ self.image_processor = GLM4VImageProcessorFast(
34
+ size={"width": image_size, "height": image_size}
35
+ )
vllm/transformers_utils/processors/pixtral.py
new
+116
@@ -0,0 +1,116 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import torch
4
+from mistral_common.protocol.instruct.chunk import ImageChunk
5
+from mistral_common.tokens.tokenizers.multimodal import ImageEncoder
6
+from PIL import Image
7
+from transformers import BatchFeature, ProcessorMixin, TensorType
8
+from transformers.audio_utils import AudioInput
9
+from transformers.image_utils import ImageInput
10
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
11
+from transformers.video_utils import VideoInput
12
+
13
+from vllm.tokenizers.mistral import MistralTokenizer
14
+
15
+
16
+class MistralCommonImageProcessor:
17
+ """
18
+ Provide a HF-compatible interface for
19
+ `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
20
+ """
21
+
22
+ def __init__(self, mm_encoder: ImageEncoder) -> None:
23
+ self.mm_encoder = mm_encoder
24
+
25
+ def __call__(
26
+ self,
27
+ images: ImageInput,
28
+ return_tensors: str | TensorType | None = None,
29
+ **kwargs,
30
+ ) -> BatchFeature:
31
+ images_lst = [images] if not isinstance(images, list) else images
32
+
33
+ images_processed = list[torch.Tensor]()
34
+
35
+ for image in images_lst:
36
+ image_inputs = self.mm_encoder(ImageChunk(image=image))
37
+ image_processed = torch.tensor(image_inputs.image)
38
+
39
+ images_processed.append(image_processed)
40
+
41
+ return BatchFeature({"images": images_processed}, tensor_type=return_tensors)
42
+
43
+ def get_number_of_image_patches(
44
+ self,
45
+ height: int,
46
+ width: int,
47
+ ) -> tuple[int, int, int]:
48
+ image = Image.new("RGB", (width, height))
49
+ ncols, nrows = self.mm_encoder._image_to_num_tokens(image)
50
+ return ncols * nrows, nrows, ncols
51
+
52
+
53
+class MistralCommonPixtralProcessor(ProcessorMixin):
54
+ attributes = ["image_processor", "tokenizer"]
55
+
56
+ def __init__(self, tokenizer: MistralTokenizer) -> None:
57
+ self.tokenizer = tokenizer.transformers_tokenizer
58
+ self.image_processor = MistralCommonImageProcessor(
59
+ tokenizer.instruct.mm_encoder
60
+ )
61
+
62
+ self._image_special_ids = self.image_processor.mm_encoder.special_ids
63
+
64
+ @property
65
+ def image_break_id(self) -> int:
66
+ return self._image_special_ids.img_break
67
+
68
+ @property
69
+ def image_token_id(self) -> int:
70
+ return self._image_special_ids.img
71
+
72
+ @property
73
+ def image_end_id(self) -> int:
74
+ return self._image_special_ids.img_end
75
+
76
+ def __call__(
77
+ self,
78
+ images: ImageInput | None = None,
79
+ text: TextInput
80
+ | PreTokenizedInput
81
+ | list[TextInput]
82
+ | list[PreTokenizedInput]
83
+ | None = None,
84
+ videos: VideoInput | None = None,
85
+ audio: AudioInput | None = None,
86
+ **kwargs,
87
+ ):
88
+ if images is None and text is None and videos is None and audio is None:
89
+ raise ValueError(
90
+ f"You need to provide at least one input to "
91
+ f"call {self.__class__.__name__}"
92
+ )
93
+
94
+ kwargs = self._merge_kwargs(
95
+ self.valid_processor_kwargs,
96
+ tokenizer_init_kwargs={},
97
+ **kwargs,
98
+ )
99
+ kwargs["text_kwargs"]["return_tensors"] = "pt"
100
+ kwargs["images_kwargs"]["return_tensors"] = None # Avoid padding issue
101
+
102
+ attribute_to_kwargs = {
103
+ "tokenizer": (text, "text_kwargs"),
104
+ "image_processor": (images, "images_kwargs"),
105
+ "video_processor": (videos, "videos_kwargs"),
106
+ "feature_extractor": (audio, "audio_kwargs"),
107
+ }
108
+ outputs = {}
109
+ for attribute_name in self.attributes:
110
+ attribute = getattr(self, attribute_name, None)
111
+ input_data, input_kwargs = attribute_to_kwargs[attribute_name]
112
+ if input_data is not None and attribute is not None:
113
+ attribute_output = attribute(input_data, **kwargs[input_kwargs])
114
+ outputs.update(attribute_output)
115
+
116
+ return BatchFeature(outputs)
vllm/transformers_utils/processors/qwen_vl.py
new
+48
@@ -0,0 +1,48 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from transformers.image_processing_utils_fast import BaseImageProcessorFast
4
+from transformers.image_utils import PILImageResampling
5
+from transformers.processing_utils import ProcessorMixin
6
+
7
+from vllm.tokenizers.qwen_vl import QwenVLTokenizer
8
+
9
+
10
+class QwenVLImageProcessorFast(BaseImageProcessorFast):
11
+ """
12
+ Port of https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
13
+ to HF Transformers.
14
+ """
15
+
16
+ resample = PILImageResampling.BICUBIC
17
+ image_mean = [0.48145466, 0.4578275, 0.40821073]
18
+ image_std = [0.26862954, 0.26130258, 0.27577711]
19
+ size = {"height": 448, "width": 448}
20
+ do_resize = True
21
+ do_rescale = True
22
+ do_normalize = True
23
+
24
+
25
+class QwenVLProcessor(ProcessorMixin):
26
+ attributes = ["image_processor", "tokenizer"]
27
+
28
+ def __init__(
29
+ self,
30
+ tokenizer: QwenVLTokenizer,
31
+ image_size: int,
32
+ ) -> None:
33
+ self.tokenizer = tokenizer
34
+ self.image_processor = QwenVLImageProcessorFast(
35
+ size={"width": image_size, "height": image_size}
36
+ )
37
+
38
+ @property
39
+ def image_start_tag(self) -> str:
40
+ return self.tokenizer.image_start_tag # type: ignore[attr-defined]
41
+
42
+ @property
43
+ def image_end_tag(self) -> str:
44
+ return self.tokenizer.image_end_tag # type: ignore[attr-defined]
45
+
46
+ @property
47
+ def image_pad_tag(self) -> str:
48
+ return self.tokenizer.image_pad_tag # type: ignore[attr-defined]
vllm/transformers_utils/processors/voxtral.py
new
+119
@@ -0,0 +1,119 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from math import ceil
5
+
6
+import numpy as np
7
+import torch
8
+from mistral_common.tokens.tokenizers.audio import AudioEncoder
9
+from transformers import BatchFeature, ProcessorMixin, TensorType
10
+from transformers.audio_utils import AudioInput
11
+from transformers.image_utils import ImageInput
12
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
13
+from transformers.video_utils import VideoInput
14
+
15
+from vllm.tokenizers.mistral import MistralTokenizer
16
+
17
+
18
+class MistralCommonFeatureExtractor:
19
+ """
20
+ Provide a HF-compatible interface for
21
+ `mistral_common.tokens.tokenizers.multimodal.AudioEncoder`.
22
+ """
23
+
24
+ def __init__(self, audio_encoder: AudioEncoder) -> None:
25
+ self.audio_encoder = audio_encoder
26
+
27
+ @property
28
+ def sampling_rate(self):
29
+ return self.audio_encoder.audio_config.sampling_rate
30
+
31
+ @property
32
+ def frame_rate(self):
33
+ return self.audio_encoder.audio_config.frame_rate
34
+
35
+ def __call__(
36
+ self,
37
+ audios: AudioInput,
38
+ return_tensors: str | TensorType | None = None,
39
+ **kwargs,
40
+ ) -> BatchFeature:
41
+ audios_lst = [audios] if not isinstance(audios, list) else audios
42
+
43
+ audios_processed = list[torch.Tensor]()
44
+
45
+ for audio in audios_lst:
46
+ audio = np.asarray(audio, dtype=np.float32).ravel()
47
+ if not self.audio_encoder.audio_config.is_streaming:
48
+ audio = self.audio_encoder.pad(audio, self.sampling_rate)
49
+
50
+ audios_processed.append(torch.tensor(audio))
51
+
52
+ return BatchFeature(
53
+ {"audio_arrays": audios_processed}, tensor_type=return_tensors
54
+ )
55
+
56
+ def get_num_audio_tokens(self, audio_length: int) -> int:
57
+ return ceil(audio_length / (self.sampling_rate // self.frame_rate))
58
+
59
+
60
+class MistralCommonVoxtralProcessor(ProcessorMixin):
61
+ attributes = ["feature_extractor", "tokenizer"]
62
+
63
+ def __init__(self, tokenizer: MistralTokenizer) -> None:
64
+ self.tokenizer = tokenizer.transformers_tokenizer
65
+ self.feature_extractor = MistralCommonFeatureExtractor(
66
+ tokenizer.instruct.audio_encoder
67
+ )
68
+
69
+ self._audio_special_ids = self.feature_extractor.audio_encoder.special_ids
70
+
71
+ @property
72
+ def audio_token_id(self) -> int:
73
+ return self._audio_special_ids.audio
74
+
75
+ @property
76
+ def begin_audio_token_id(self) -> int:
77
+ return self._audio_special_ids.begin_audio
78
+
79
+ def __call__(
80
+ self,
81
+ images: ImageInput | None = None,
82
+ text: TextInput
83
+ | PreTokenizedInput
84
+ | list[TextInput]
85
+ | list[PreTokenizedInput]
86
+ | None = None,
87
+ videos: VideoInput | None = None,
88
+ audio: AudioInput | None = None,
89
+ **kwargs,
90
+ ):
91
+ if images is None and text is None and videos is None and audio is None:
92
+ raise ValueError(
93
+ f"You need to provide at least one input to "
94
+ f"call {self.__class__.__name__}"
95
+ )
96
+
97
+ kwargs = self._merge_kwargs(
98
+ self.valid_processor_kwargs,
99
+ tokenizer_init_kwargs={},
100
+ **kwargs,
101
+ )
102
+ kwargs["text_kwargs"]["return_tensors"] = "pt"
103
+ kwargs["audio_kwargs"]["return_tensors"] = None # Avoid padding issue
104
+
105
+ attribute_to_kwargs = {
106
+ "tokenizer": (text, "text_kwargs"),
107
+ "image_processor": (images, "images_kwargs"),
108
+ "video_processor": (videos, "videos_kwargs"),
109
+ "feature_extractor": (audio, "audio_kwargs"),
110
+ }
111
+ outputs = {}
112
+ for attribute_name in self.attributes:
113
+ attribute = getattr(self, attribute_name, None)
114
+ input_data, input_kwargs = attribute_to_kwargs[attribute_name]
115
+ if input_data is not None and attribute is not None:
116
+ attribute_output = attribute(input_data, **kwargs[input_kwargs])
117
+ outputs.update(attribute_output)
118
+
119
+ return BatchFeature(outputs)