A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Doc] [2/N] Add Fuyu E2E example for multimodal processor (#13331)
Cyrus Leung committed
Feb 15, 2025 at 23:06 UTC
367cb8ce8c0486e22941cf8990902fc5ec992ec1
2 files changed
+529
-31
docs/source/contributing/model/multimodal.md
+525
-29
@@ -262,6 +262,255 @@ def get_mm_max_tokens_per_item(
262
Our [actual code](gh-file:vllm/model_executor/models/llava.py) is more abstracted to support vision encoders other than CLIP.
263
:::
264
265
+::::
266
+
267
+::::{tab-item} Non-consecutive feature tokens: Fuyu
268
+:sync: fuyu
269
+
270
+Looking at the code of HF's `FuyuForCausalLM`:
271
+
272
+```python
273
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/modeling_fuyu.py#L311-L322
274
+if image_patches is not None and past_key_values is None:
275
+ patch_embeddings = [
276
+ self.vision_embed_tokens(patch.to(self.vision_embed_tokens.weight.dtype))
277
+ .squeeze(0)
278
+ .to(inputs_embeds.device)
279
+ for patch in image_patches
280
+ ]
281
+ inputs_embeds = self.gather_continuous_embeddings(
282
+ word_embeddings=inputs_embeds,
283
+ continuous_embeddings=patch_embeddings,
284
+ image_patch_input_indices=image_patches_indices,
285
+ )
286
+```
287
+
288
+The number of placeholder feature tokens for the `i`th item in the batch is `patch_embeddings[i].shape[0]`,
289
+which is the same as `image_patches[i].shape[0]`, i.e. `num_total_patches`.
290
+
291
+Unlike LLaVA, Fuyu does not define the number of patches inside the modeling file. Where can we get more information?
292
+Considering that the model input comes from the output of `FuyuProcessor`, let's **look at the preprocessing files**.
293
+
294
+The image outputs are obtained by calling `FuyuImageProcessor.preprocess` and then
295
+`FuyuImageProcessor.preprocess_with_tokenizer_info` inside `FuyuProcessor`.
296
+
297
+In `FuyuImageProcessor.preprocess`, the images are resized and padded to the target `FuyuImageProcessor.size`,
298
+returning the dimensions after resizing (but before padding) as metadata.
299
+
300
+```python
301
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L541-L544
302
+image_encoding = self.image_processor.preprocess(images, **output_kwargs["images_kwargs"])
303
+batch_images = image_encoding["images"]
304
+image_unpadded_heights = image_encoding["image_unpadded_heights"]
305
+image_unpadded_widths = image_encoding["image_unpadded_widths"]
306
+
307
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L480-L
308
+if do_resize:
309
+ batch_images = [
310
+ [self.resize(image, size=size, input_data_format=input_data_format) for image in images]
311
+ for images in batch_images
312
+ ]
313
+
314
+image_sizes = [get_image_size(images[0], channel_dim=input_data_format) for images in batch_images]
315
+image_unpadded_heights = [[image_size[0]] for image_size in image_sizes]
316
+image_unpadded_widths = [[image_size[1]] for image_size in image_sizes]
317
+
318
+if do_pad:
319
+ batch_images = [
320
+ [
321
+ self.pad_image(
322
+ image,
323
+ size=size,
324
+ mode=padding_mode,
325
+ constant_values=padding_value,
326
+ input_data_format=input_data_format,
327
+ )
328
+ for image in images
329
+ ]
330
+ for images in batch_images
331
+ ]
332
+```
333
+
334
+In `FuyuImageProcessor.preprocess_with_tokenizer_info`, the images are split into patches based on this metadata:
335
+
336
+```python
337
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L417-L425
338
+model_image_input = self.image_processor.preprocess_with_tokenizer_info(
339
+ image_input=tensor_batch_images,
340
+ image_present=image_present,
341
+ image_unpadded_h=image_unpadded_heights,
342
+ image_unpadded_w=image_unpadded_widths,
343
+ image_placeholder_id=image_placeholder_id,
344
+ image_newline_id=image_newline_id,
345
+ variable_sized=True,
346
+)
347
+
348
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L638-L658
349
+image_height, image_width = image.shape[1], image.shape[2]
350
+if variable_sized: # variable_sized=True
351
+ new_h = min(
352
+ image_height,
353
+ math.ceil(image_unpadded_h[batch_index, subseq_index] / patch_height) * patch_height,
354
+ )
355
+ new_w = min(
356
+ image_width,
357
+ math.ceil(image_unpadded_w[batch_index, subseq_index] / patch_width) * patch_width,
358
+ )
359
+ image = image[:, :new_h, :new_w]
360
+ image_height, image_width = new_h, new_w
361
+
362
+num_patches = self.get_num_patches(image_height=image_height, image_width=image_width)
363
+tensor_of_image_ids = torch.full(
364
+ [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device
365
+)
366
+patches = self.patchify_image(image=image.unsqueeze(0)).squeeze(0)
367
+assert num_patches == patches.shape[0]
368
+```
369
+
370
+The number of patches is in turn defined by `FuyuImageProcessor.get_num_patches`:
371
+
372
+```python
373
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L552-L562
374
+patch_size = patch_size if patch_size is not None else self.patch_size
375
+patch_height, patch_width = self.patch_size["height"], self.patch_size["width"]
376
+
377
+if image_height % patch_height != 0:
378
+ raise ValueError(f"{image_height=} must be divisible by {patch_height}")
379
+if image_width % patch_width != 0:
380
+ raise ValueError(f"{image_width=} must be divisible by {patch_width}")
381
+
382
+num_patches_per_dim_h = image_height // patch_height
383
+num_patches_per_dim_w = image_width // patch_width
384
+num_patches = num_patches_per_dim_h * num_patches_per_dim_w
385
+```
386
+
387
+We can calculate this in vLLM using this code:
388
+
389
+```python
390
+def get_num_image_patches(
391
+ self,
392
+ *,
393
+ image_width: int,
394
+ image_height: int,
395
+) -> int:
396
+ image_processor = self.get_image_processor()
397
+ target_width = image_processor.size["width"]
398
+ target_height = image_processor.size["height"]
399
+ patch_width = image_processor.patch_size["width"]
400
+ patch_height = image_processor.patch_size["height"]
401
+
402
+ if not (image_width <= target_width and image_height <= target_height):
403
+ height_scale_factor = target_height / image_height
404
+ width_scale_factor = target_width / image_width
405
+ optimal_scale_factor = min(height_scale_factor, width_scale_factor)
406
+
407
+ image_height = int(image_height * optimal_scale_factor)
408
+ image_width = int(image_width * optimal_scale_factor)
409
+
410
+ ncols = math.ceil(image_width / patch_width)
411
+ nrows = math.ceil(image_height / patch_height)
412
+ return ncols * nrows
413
+```
414
+
415
+These image patches correspond to placeholder tokens (`|SPEAKER|`). However, the processor also
416
+inserts newline tokens (`|NEWLINE|`) as shown here:
417
+
418
+```python
419
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L654-L670
420
+tensor_of_image_ids = torch.full(
421
+ [num_patches], image_placeholder_id, dtype=torch.int32, device=image_input.device
422
+)
423
+patches = self.patchify_image(image=image.unsqueeze(0)).squeeze(0)
424
+assert num_patches == patches.shape[0]
425
+
426
+if variable_sized:
427
+ # Now terminate each line with |NEWLINE|.
428
+ tensor_of_image_ids = tensor_of_image_ids.reshape(-1, image_width // patch_width)
429
+ newline_ids = torch.full(
430
+ [tensor_of_image_ids.shape[0], 1],
431
+ image_newline_id,
432
+ dtype=torch.int32,
433
+ device=image_input.device,
434
+ )
435
+ tensor_of_image_ids = torch.cat([tensor_of_image_ids, newline_ids], dim=1)
436
+ tensor_of_image_ids = tensor_of_image_ids.reshape(-1)
437
+```
438
+
439
+So, the layout of tokens for an image is:
440
+
441
+```
442
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
443
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
444
+...
445
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
446
+```
447
+
448
+This makes the placeholder tokens non-consecutive in the prompt.
449
+Since vLLM requires the feature tokens to be consecutive, **we also treat the newline tokens as feature tokens**.
450
+
451
+So overall, the total number of feature tokens is
452
+
453
+```python
454
+def get_num_image_tokens(
455
+ self,
456
+ *,
457
+ image_width: int,
458
+ image_height: int,
459
+) -> int:
460
+ image_processor = self.get_image_processor()
461
+ target_width = image_processor.size["width"]
462
+ target_height = image_processor.size["height"]
463
+ patch_width = image_processor.patch_size["width"]
464
+ patch_height = image_processor.patch_size["height"]
465
+
466
+ if not (image_width <= target_width and image_height <= target_height):
467
+ height_scale_factor = target_height / image_height
468
+ width_scale_factor = target_width / image_width
469
+ optimal_scale_factor = min(height_scale_factor, width_scale_factor)
470
+
471
+ image_height = int(image_height * optimal_scale_factor)
472
+ image_width = int(image_width * optimal_scale_factor)
473
+
474
+ ncols = math.ceil(image_width / patch_width)
475
+ nrows = math.ceil(image_height / patch_height)
476
+ return (ncols + 1) * nrows
477
+```
478
+
479
+To calculate the maximum number of image tokens, recall that input images are first resized
480
+to fit within `image_processor.size`. The maximum possible dimensions of the image before
481
+being converted into patches is therefore equal to `image_processor.size`.
482
+
483
+```python
484
+def get_image_size_with_most_features(self) -> ImageSize:
485
+ image_processor = self.get_image_processor()
486
+ return ImageSize(width=image_processor.size["width"],
487
+ height=image_processor.size["height"])
488
+
489
+def get_max_image_tokens(self) -> int:
490
+ target_width, target_height = self.get_image_size_with_most_features()
491
+
492
+ return self.get_num_image_tokens(
493
+ image_width=target_width,
494
+ image_height=target_height,
495
+ )
496
+```
497
+
498
+And thus, we can override the method as:
499
+
500
+```python
501
+def get_mm_max_tokens_per_item(
502
+ self,
503
+ seq_len: int,
504
+ mm_counts: Mapping[str, int],
505
+) -> Mapping[str, int]:
506
+ return {"image": self.get_max_image_tokens()}
507
+```
508
+
509
+:::{note}
510
+Our [actual code](gh-file:vllm/model_executor/models/fuyu.py) returns `ncols` and `nrows` directly instead of the total token count.
511
+This is because `ncols` and `nrows` are used to specify the layout of the feature tokens (as shown in Step 4 of this guide).
512
+:::
513
+
514
::::
515
:::::
516
@@ -282,7 +531,8 @@ on the code for {meth}`~vllm.multimodal.processing.BaseProcessingInfo.get_mm_max
531
::::{tab-set}
532
:::{tab-item} Basic example: LLaVA
533
:sync: llava
285
-Making use of the `get_image_size_with_most_features` method implemented in the previous section:
534
+
535
+Making use of the `get_image_size_with_most_features` method implemented in Step 2:
536
537
```python
538
def get_dummy_processor_inputs(
@@ -312,6 +562,39 @@ def get_dummy_processor_inputs(
562
```
563
564
:::
565
+
566
+:::{tab-item} No input placeholders: Fuyu
567
+:sync: fuyu
568
+
569
+Fuyu does not expect image placeholders in the inputs to HF processor, so
570
+the dummy prompt text is empty regardless of the number of images.
571
+Otherwise, the logic of this method is very similar to LLaVA:
572
+
573
+```python
574
+def get_dummy_processor_inputs(
575
+ self,
576
+ seq_len: int,
577
+ mm_counts: Mapping[str, int],
578
+) -> ProcessorInputs:
579
+ target_width, target_height = \
580
+ self.info.get_image_size_with_most_features()
581
+ num_images = mm_counts.get("image", 0)
582
+
583
+ mm_data = {
584
+ "image":
585
+ self._get_dummy_images(width=target_width,
586
+ height=target_height,
587
+ num_images=num_images)
588
+ }
589
+
590
+ return ProcessorInputs(
591
+ prompt_text="",
592
+ mm_data=mm_data,
593
+ )
594
+```
595
+
596
+:::
597
+
598
::::
599
600
## 4. Specify processing details
@@ -325,40 +608,28 @@ to fill in the missing details about HF processing.
608
609
### Multi-modal fields
610
328
-Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` to
611
+Override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` to
612
return a schema of the tensors outputted by the HF processor that are related to the input multi-modal items.
613
614
:::::{tab-set}
615
::::{tab-item} Basic example: LLaVA
616
:sync: llava
617
335
-Looking at the model's `forward` method:
618
+The output of `CLIPImageProcessor` is a simple tensor with shape
619
+`(num_images, num_channels, image_height, image_width)`:
620
621
```python
338
-# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/llava/modeling_llava.py#L387-L404
339
-def forward(
340
- self,
341
- input_ids: torch.LongTensor = None,
342
- pixel_values: torch.FloatTensor = None,
343
- attention_mask: Optional[torch.Tensor] = None,
344
- position_ids: Optional[torch.LongTensor] = None,
345
- past_key_values: Optional[List[torch.FloatTensor]] = None,
346
- inputs_embeds: Optional[torch.FloatTensor] = None,
347
- vision_feature_layer: Optional[int] = None,
348
- vision_feature_select_strategy: Optional[str] = None,
349
- labels: Optional[torch.LongTensor] = None,
350
- use_cache: Optional[bool] = None,
351
- output_attentions: Optional[bool] = None,
352
- output_hidden_states: Optional[bool] = None,
353
- return_dict: Optional[bool] = None,
354
- cache_position: Optional[torch.LongTensor] = None,
355
- num_logits_to_keep: int = 0,
356
-) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
357
-```
358
-
359
-The only related keyword argument is `pixel_values` which directly corresponds to input images.
360
-The shape of `pixel_values` is `(N, C, H, W)` where `N` is the number of images.
361
-So, we override the method as follows:
622
+# https://github.com/huggingface/transformers/blob/v4.47.1/src/transformers/models/clip/image_processing_clip.py#L339-L345
623
+images = [
624
+ to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)
625
+ for image in all_images
626
+]
627
+
628
+data = {"pixel_values": images}
629
+return BatchFeature(data=data, tensor_type=return_tensors)
630
+```
631
+
632
+So, we override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` as follows:
633
634
```python
635
def _get_mm_fields_config(
@@ -377,11 +648,83 @@ pre-computed image embeddings, which can be passed to be model via the `image_em
648
:::
649
650
::::
651
+
652
+::::{tab-item} With postprocessing: Fuyu
653
+:sync: fuyu
654
+
655
+The `image_patches` output of `FuyuImageProcessor.preprocess_with_tokenizer_info` concatenates
656
+the patches from each image belonging to an item in the batch:
657
+
658
+```python
659
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/image_processing_fuyu.py#L673-L679
660
+ image_input_ids.append(tensor_of_image_ids)
661
+ image_patches.append(patches)
662
+ else:
663
+ image_input_ids.append(torch.tensor([], dtype=torch.int32, device=image_input.device))
664
+
665
+batch_image_input_ids.append(image_input_ids)
666
+batch_image_patches.append(image_patches)
667
+```
668
+
669
+The shape of `image_patches` outputted by `FuyuImageProcessor` is therefore
670
+`(1, num_images, num_patches, patch_width * patch_height * num_channels)`.
671
+
672
+In order to support the use of {func}`MultiModalFieldConfig.batched` like in LLaVA,
673
+we remove the extra batch dimension by overriding {meth}`BaseMultiModalProcessor._call_hf_processor`:
674
+
675
+```python
676
+def _call_hf_processor(
677
+ self,
678
+ prompt: str,
679
+ mm_data: Mapping[str, object],
680
+ mm_kwargs: Mapping[str, object],
681
+) -> BatchFeature:
682
+ processed_outputs = super()._call_hf_processor(
683
+ prompt=prompt,
684
+ mm_data=mm_data,
685
+ mm_kwargs=mm_kwargs,
686
+ )
687
+
688
+ image_patches = processed_outputs.get("image_patches")
689
+ if image_patches is not None:
690
+ images = mm_data["images"]
691
+ assert isinstance(images, list)
692
+
693
+ # Original output: (1, num_images, Pn, Px * Py * C)
694
+ # New output: (num_images, Pn, Px * Py * C)
695
+ assert (isinstance(image_patches, list)
696
+ and len(image_patches) == 1)
697
+ assert (isinstance(image_patches[0], torch.Tensor)
698
+ and len(image_patches[0]) == len(images))
699
+
700
+ processed_outputs["image_patches"] = image_patches[0]
701
+
702
+ return processed_outputs
703
+```
704
+
705
+:::{note}
706
+Our [actual code](gh-file:vllm/model_executor/models/fuyu.py) has special handling
707
+for text-only inputs to prevent unnecessary warnings from HF processor.
708
+:::
709
+
710
+This lets us override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config` as follows:
711
+
712
+```python
713
+def _get_mm_fields_config(
714
+ self,
715
+ hf_inputs: BatchFeature,
716
+ hf_processor_mm_kwargs: Mapping[str, object],
717
+) -> Mapping[str, MultiModalFieldConfig]:
718
+ return dict(image_patches=MultiModalFieldConfig.batched("image"))
719
+```
720
+
721
+::::
722
+
723
:::::
724
725
### Prompt replacements
726
384
-Override {class}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` to
727
+Override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` to
728
return a list of {class}`~vllm.multimodal.processing.PromptReplacement` instances.
729
730
Each {class}`~vllm.multimodal.processing.PromptReplacement` instance specifies a find-and-replace
@@ -402,7 +745,7 @@ for sample in text:
745
```
746
747
It simply repeats each input `image_token` a number of times equal to the number of placeholder feature tokens (`num_image_tokens`).
405
-Based on this, we override the method as follows:
748
+Based on this, we override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` as follows:
749
750
```python
751
def _get_prompt_replacements(
@@ -435,6 +778,159 @@ def _get_prompt_replacements(
778
```
779
780
:::
781
+
782
+:::{tab-item} Handling additional tokens: Fuyu
783
+:sync: fuyu
784
+
785
+Recall the layout of feature tokens from Step 2:
786
+
787
+```
788
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
789
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
790
+...
791
+|SPEAKER||SPEAKER|...|SPEAKER||NEWLINE|
792
+```
793
+
794
+We define a helper function to return `ncols` and `nrows` directly:
795
+
796
+```python
797
+def get_image_feature_grid_size(
798
+ self,
799
+ *,
800
+ image_width: int,
801
+ image_height: int,
802
+) -> tuple[int, int]:
803
+ image_processor = self.get_image_processor()
804
+ target_width = image_processor.size["width"]
805
+ target_height = image_processor.size["height"]
806
+ patch_width = image_processor.patch_size["width"]
807
+ patch_height = image_processor.patch_size["height"]
808
+
809
+ if not (image_width <= target_width and image_height <= target_height):
810
+ height_scale_factor = target_height / image_height
811
+ width_scale_factor = target_width / image_width
812
+ optimal_scale_factor = min(height_scale_factor, width_scale_factor)
813
+
814
+ image_height = int(image_height * optimal_scale_factor)
815
+ image_width = int(image_width * optimal_scale_factor)
816
+
817
+ ncols = math.ceil(image_width / patch_width)
818
+ nrows = math.ceil(image_height / patch_height)
819
+ return ncols, nrows
820
+```
821
+
822
+Based on this, we can initially define our replacement tokens as:
823
+
824
+```python
825
+def get_replacement(item_idx: int):
826
+ images = mm_items.get_items("image", ImageProcessorItems)
827
+ image_size = images.get_image_size(item_idx)
828
+
829
+ ncols, nrows = self.info.get_image_feature_grid_size(
830
+ image_width=image_size.width,
831
+ image_height=image_size.height,
832
+ )
833
+
834
+ # `_IMAGE_TOKEN_ID` corresponds to `|SPEAKER|`
835
+ # `_NEWLINE_TOKEN_ID` corresponds to `|NEWLINE|`
836
+ return ([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows
837
+```
838
+
839
+However, this is not entirely correct. After `FuyuImageProcessor.preprocess_with_tokenizer_info` is called,
840
+a BOS token (`<s>`) is also added to the promopt:
841
+
842
+```python
843
+# https://github.com/huggingface/transformers/blob/v4.48.3/src/transformers/models/fuyu/processing_fuyu.py#L417-L435
844
+model_image_input = self.image_processor.preprocess_with_tokenizer_info(
845
+ image_input=tensor_batch_images,
846
+ image_present=image_present,
847
+ image_unpadded_h=image_unpadded_heights,
848
+ image_unpadded_w=image_unpadded_widths,
849
+ image_placeholder_id=image_placeholder_id,
850
+ image_newline_id=image_newline_id,
851
+ variable_sized=True,
852
+)
853
+prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch(
854
+ tokenizer=self.tokenizer,
855
+ prompts=prompts,
856
+ scale_factors=scale_factors,
857
+ max_tokens_to_generate=self.max_tokens_to_generate,
858
+ max_position_embeddings=self.max_position_embeddings,
859
+ add_BOS=True,
860
+ add_beginning_of_answer_token=True,
861
+)
862
+```
863
+
864
+To accommodate this, instead of a string you can return an instance of `PromptReplacementDetails`
865
+with different `full` and `feature` attributes:
866
+
867
+```python
868
+hf_config = self.info.get_hf_config()
869
+bos_token_id = hf_config.bos_token_id # `<s>`
870
+assert isinstance(bos_token_id, int)
871
+
872
+def get_replacement_fuyu(item_idx: int):
873
+ images = mm_items.get_items("image", ImageProcessorItems)
874
+ image_size = images.get_image_size(item_idx)
875
+
876
+ ncols, nrows = self.info.get_image_feature_grid_size(
877
+ image_width=image_size.width,
878
+ image_height=image_size.height,
879
+ )
880
+ image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
881
+ [_NEWLINE_TOKEN_ID]) * nrows
882
+
883
+ return PromptReplacementDetails(
884
+ full=image_tokens + [bos_token_id],
885
+ features=image_tokens,
886
+ )
887
+```
888
+
889
+Finally, noticing that the HF processor removes the `|ENDOFTEXT|` token from the tokenized prompt,
890
+we can search for it to conduct the replacement at the start of the string:
891
+
892
+```python
893
+def _get_prompt_replacements(
894
+ self,
895
+ mm_items: MultiModalDataItems,
896
+ hf_processor_mm_kwargs: Mapping[str, object],
897
+ out_mm_kwargs: MultiModalKwargs,
898
+) -> list[PromptReplacement]:
899
+ hf_config = self.info.get_hf_config()
900
+ bos_token_id = hf_config.bos_token_id
901
+ assert isinstance(bos_token_id, int)
902
+
903
+ tokenizer = self.info.get_tokenizer()
904
+ eot_token_id = tokenizer.bos_token_id
905
+ assert isinstance(eot_token_id, int)
906
+
907
+ def get_replacement_fuyu(item_idx: int):
908
+ images = mm_items.get_items("image", ImageProcessorItems)
909
+ image_size = images.get_image_size(item_idx)
910
+
911
+ ncols, nrows = self.info.get_image_feature_grid_size(
912
+ image_width=image_size.width,
913
+ image_height=image_size.height,
914
+ )
915
+ image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
916
+ [_NEWLINE_TOKEN_ID]) * nrows
917
+
918
+ return PromptReplacementDetails(
919
+ full=image_tokens + [bos_token_id],
920
+ features=image_tokens,
921
+ )
922
+
923
+ return [
924
+ PromptReplacement(
925
+ modality="image",
926
+ target=[eot_token_id],
927
+ replacement=get_replacement_fuyu,
928
+ )
929
+ ]
930
+```
931
+
932
+:::
933
+
934
::::
935
936
## 5. Register processor-related classes
vllm/model_executor/models/fuyu.py
+4
-2
@@ -104,6 +104,8 @@ class FuyuProcessingInfo(BaseProcessingInfo):
104
image_processor = self.get_image_processor()
105
target_width = image_processor.size["width"]
106
target_height = image_processor.size["height"]
107
+ patch_width = image_processor.patch_size["width"]
108
+ patch_height = image_processor.patch_size["height"]
109
110
if not (image_width <= target_width and image_height <= target_height):
111
height_scale_factor = target_height / image_height
@@ -113,8 +115,8 @@ class FuyuProcessingInfo(BaseProcessingInfo):
115
image_height = int(image_height * optimal_scale_factor)
116
image_width = int(image_width * optimal_scale_factor)
117
116
- ncols = math.ceil(image_width / 30)
117
- nrows = math.ceil(image_height / 30)
118
+ ncols = math.ceil(image_width / patch_width)
119
+ nrows = math.ceil(image_height / patch_height)
120
return ncols, nrows
121
122
def get_image_size_with_most_features(self) -> ImageSize: