A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 20:00 UTC
Support MiniCPMV batched preprocessing (#44609)
Signed-off-by: Yan Ma <yan.ma@intel.com>
Yan Ma committed
Jun 5, 2026 at 23:05 UTC
7f003a1285af4bca511606b8731171347300f893
1 file changed
+63
-56
vllm/transformers_utils/processors/minicpmv.py
+63
-56
@@ -72,8 +72,8 @@ class MiniCPMVProcessor(ProcessorMixin):
72
) -> MiniCPMVBatchFeature:
73
"""Run the vendored MiniCPMV processor on a (text, images) pair.
74
75
- Only single-sample input is currently supported; batched input is
76
- coming soon. ``images`` is forwarded to the underlying image
75
+ Batched inputs are supported following the upstream MiniCPM-V
76
+ processor flow. ``images`` is forwarded to the underlying image
77
processor and ``text`` is tokenized with image placeholders
78
replaced by the appropriate slice tokens. Returns a
79
``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when
@@ -194,7 +194,7 @@ class MiniCPMVProcessor(ProcessorMixin):
194
image_end_tokens.unsqueeze(-1),
195
]
196
)
197
- return input_ids.unsqueeze(0), image_bounds
197
+ return input_ids, image_bounds
198
199
def _convert_images_texts_to_inputs(
200
self,
@@ -220,23 +220,41 @@ class MiniCPMVProcessor(ProcessorMixin):
220
image_sizes = images["image_sizes"]
221
tgt_sizes = images["tgt_sizes"]
222
223
- image_tags = regex.findall(pattern, texts)
224
- assert len(image_tags) == len(image_sizes[0])
225
- text_chunks = texts.split(pattern)
226
- final_texts = ""
227
- for i in range(len(image_tags)):
228
- placeholder = self.image_processor.get_slice_image_placeholder(
229
- image_sizes[0][i]
230
- )
231
- final_texts = final_texts + text_chunks[i] + placeholder
232
- final_texts += text_chunks[-1]
233
- input_ids, image_bounds = self._convert(final_texts, max_length)
223
+ if isinstance(texts, str):
224
+ texts = [texts]
225
+
226
+ input_ids_list = []
227
+ image_bounds_list = []
228
+
229
+ for index, text in enumerate(texts):
230
+ image_tags = regex.findall(pattern, text)
231
+ assert len(image_tags) == len(image_sizes[index])
232
+ text_chunks = text.split(pattern)
233
+ final_text = ""
234
+ for i in range(len(image_tags)):
235
+ placeholder = self.image_processor.get_slice_image_placeholder(
236
+ image_sizes[index][i]
237
+ )
238
+ final_text = final_text + text_chunks[i] + placeholder
239
+ final_text += text_chunks[-1]
240
+ input_ids, image_bounds = self._convert(final_text, max_length)
241
+ input_ids_list.append(input_ids)
242
+ image_bounds_list.append(image_bounds)
243
+
244
+ padded_input_ids, padding_lengths = self.pad(
245
+ input_ids_list,
246
+ padding_side="left",
247
+ )
248
+ for i, length in enumerate(padding_lengths):
249
+ image_bounds_list[i] = image_bounds_list[i] + length
250
+
251
return MiniCPMVBatchFeature(
252
data={
236
- "input_ids": input_ids,
253
+ "input_ids": padded_input_ids,
254
+ "attention_mask": padded_input_ids.ne(0),
255
"pixel_values": images_val,
256
"image_sizes": image_sizes,
239
- "image_bound": [image_bounds],
257
+ "image_bound": image_bounds_list,
258
"tgt_sizes": tgt_sizes,
259
}
260
)
@@ -249,42 +267,36 @@ class MiniCPMVProcessor(ProcessorMixin):
267
image_processor_input_names = self.image_processor.model_input_names
268
return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
269
252
- def pad(
253
- self,
254
- orig_items,
255
- key,
256
- max_length=None,
257
- padding_value=0,
258
- padding_side="left",
259
- ):
260
- if not orig_items:
261
- return torch.empty(0)
270
+ # Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py.
271
+ def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"):
272
+ if not inputs:
273
+ return torch.empty(0), []
274
275
items = []
264
- if isinstance(orig_items[0][key], list):
265
- assert isinstance(orig_items[0][key][0], torch.Tensor)
266
- for it in orig_items:
267
- for tr in it[key]:
268
- items.append({key: tr})
276
+ if isinstance(inputs[0], list):
277
+ assert isinstance(inputs[0][0], torch.Tensor)
278
+ for it in inputs:
279
+ for tr in it:
280
+ items.append(tr)
281
else:
270
- assert isinstance(orig_items[0][key], torch.Tensor)
271
- items = orig_items
282
+ assert isinstance(inputs[0], torch.Tensor)
283
+ items = inputs
284
285
batch_size = len(items)
274
- shape = items[0][key].shape
286
+ shape = items[0].shape
287
dim = len(shape)
276
- assert dim <= 3
288
+ assert dim <= 2
289
if max_length is None:
290
max_length = 0
279
- max_length = max(max_length, max(item[key].shape[-1] for item in items))
280
- min_length = min(item[key].shape[-1] for item in items)
281
- dtype = items[0][key].dtype
291
+ max_length = max(max_length, max(item.shape[-1] for item in items))
292
+ min_length = min(item.shape[-1] for item in items)
293
+ dtype = items[0].dtype
294
283
- if dim == 1:
284
- return torch.cat([item[key] for item in items], dim=0)
285
- elif dim == 2:
295
+ if dim == 0:
296
+ return torch.stack([item for item in items], dim=0), [0]
297
+ elif dim == 1:
298
if max_length == min_length:
287
- return torch.cat([item[key] for item in items], dim=0)
299
+ return torch.stack([item for item in items], dim=0), [0] * batch_size
300
tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value
301
else:
302
tensor = (
@@ -292,23 +304,18 @@ class MiniCPMVProcessor(ProcessorMixin):
304
+ padding_value
305
)
306
307
+ padding_lengths = []
308
for i, item in enumerate(items):
296
- tensor_to_pad = item[key]
297
- if tensor_to_pad.shape[0] != 1:
298
- raise ValueError(
299
- f"Expected leading batch size of 1 for padding, "
300
- f"but got shape {tensor_to_pad.shape}"
301
- )
302
- squeezed = tensor_to_pad.squeeze(0)
303
- if dim == 2:
309
+ if dim == 1:
310
if padding_side == "left":
305
- tensor[i, -squeezed.shape[0] :] = squeezed.clone()
311
+ tensor[i, -len(item) :] = item.clone()
312
else:
307
- tensor[i, : squeezed.shape[0]] = squeezed.clone()
308
- elif dim == 3:
313
+ tensor[i, : len(item)] = item.clone()
314
+ elif dim == 2:
315
if padding_side == "left":
310
- tensor[i, -squeezed.shape[0] :, :] = squeezed.clone()
316
+ tensor[i, -len(item) :, :] = item.clone()
317
else:
312
- tensor[i, : squeezed.shape[0], :] = squeezed.clone()
318
+ tensor[i, : len(item), :] = item.clone()
319
+ padding_lengths.append(tensor.shape[-1] - len(item))
320
314
- return tensor
321
+ return tensor, padding_lengths