A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Refactor]: Use M-RoPE interface directly while defining model class instead of maintaining model specific M-RoPE implementation in mrope.py (#24172)
Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com> Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: wwl2755 <wangwenlong2755@gmail.com>
dsinghvi committed
Oct 11, 2025 at 12:51 UTC
727144bed10ffd465e37d47a5a60747efc15368b
9 files changed
+974
-1051
vllm/model_executor/layers/rotary_embedding/mrope.py
-1015
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-import itertools
4
from typing import Optional, Union
5
6
import numpy as np
@@ -411,969 +410,6 @@ class MRotaryEmbedding(RotaryEmbedding):
410
411
return llm_positions.tolist(), mrope_position_delta
412
414
- @classmethod
415
- def get_input_positions_tensor(
416
- cls,
417
- input_tokens: list[int],
418
- hf_config: PretrainedConfig,
419
- image_grid_thw: Union[list[list[int]], torch.Tensor],
420
- video_grid_thw: Union[list[list[int]], torch.Tensor],
421
- second_per_grid_ts: list[float],
422
- context_len: int = 0,
423
- seq_len: Optional[int] = None,
424
- audio_feature_lengths: Optional[torch.Tensor] = None,
425
- use_audio_in_video: bool = False,
426
- ) -> tuple[torch.Tensor, int]:
427
- from vllm.transformers_utils.config import thinker_uses_mrope
428
-
429
- if thinker_uses_mrope(hf_config) and hf_config.model_type == "qwen2_5_omni":
430
- return cls._omni_get_input_positions_tensor(
431
- input_tokens=input_tokens,
432
- hf_config=hf_config,
433
- image_grid_thw=image_grid_thw,
434
- video_grid_thw=video_grid_thw,
435
- second_per_grid_ts=second_per_grid_ts,
436
- context_len=context_len,
437
- seq_len=seq_len,
438
- audio_feature_lengths=audio_feature_lengths,
439
- use_audio_in_video=use_audio_in_video,
440
- )
441
- elif hf_config.model_type in ["glm4v", "glm4v_moe"]:
442
- return cls._glm4v_get_input_positions_tensor(
443
- input_tokens=input_tokens,
444
- hf_config=hf_config,
445
- image_grid_thw=image_grid_thw,
446
- video_grid_thw=video_grid_thw,
447
- context_len=context_len,
448
- seq_len=seq_len,
449
- )
450
- elif hf_config.model_type in ["qwen3_vl", "qwen3_vl_moe"]:
451
- return cls._qwen3vl_get_input_positions_tensor(
452
- input_tokens=input_tokens,
453
- hf_config=hf_config,
454
- image_grid_thw=image_grid_thw,
455
- video_grid_thw=video_grid_thw,
456
- context_len=context_len,
457
- seq_len=seq_len,
458
- )
459
- elif hf_config.model_type in ["ernie4_5_moe_vl", "ernie4_5_vl"]:
460
- return cls._ernie_get_input_positions_tensor(
461
- input_tokens=input_tokens,
462
- hf_config=hf_config,
463
- image_grid_thw=image_grid_thw,
464
- video_grid_thw=video_grid_thw,
465
- context_len=context_len,
466
- seq_len=seq_len,
467
- )
468
- elif "KeyeVL1_5" in hf_config.model_type:
469
- return cls._keye_get_input_positions_tensor(
470
- input_tokens=input_tokens,
471
- hf_config=hf_config,
472
- image_grid_thw=image_grid_thw,
473
- video_grid_thw=video_grid_thw,
474
- context_len=context_len,
475
- seq_len=seq_len,
476
- )
477
- else:
478
- return cls._vl_get_input_positions_tensor(
479
- input_tokens=input_tokens,
480
- hf_config=hf_config,
481
- image_grid_thw=image_grid_thw,
482
- video_grid_thw=video_grid_thw,
483
- second_per_grid_ts=second_per_grid_ts,
484
- context_len=context_len,
485
- seq_len=seq_len,
486
- )
487
-
488
- @classmethod
489
- def _glm4v_get_input_positions_tensor(
490
- cls,
491
- input_tokens: list[int],
492
- hf_config: PretrainedConfig,
493
- image_grid_thw: Union[list[list[int]], torch.Tensor],
494
- video_grid_thw: Union[list[list[int]], torch.Tensor],
495
- context_len: int = 0,
496
- seq_len: Optional[int] = None,
497
- ) -> tuple[torch.Tensor, int]:
498
- """Get mrope input positions and delta value for GLM4V."""
499
-
500
- image_token_id = hf_config.image_token_id
501
- video_start_token_id = hf_config.video_start_token_id
502
- video_end_token_id = hf_config.video_end_token_id
503
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
504
- llm_pos_ids_list: list = []
505
-
506
- if not (image_grid_thw is None and video_grid_thw is None):
507
- if isinstance(image_grid_thw, torch.Tensor):
508
- image_grid_thw = image_grid_thw.tolist()
509
-
510
- input_token_type: list[str] = []
511
- video_check_flg = False
512
- for token in input_tokens:
513
- if token == video_start_token_id:
514
- video_check_flg = True
515
- elif token == video_end_token_id:
516
- video_check_flg = False
517
-
518
- if (token == image_token_id) and (video_check_flg is False):
519
- input_token_type.append("image")
520
- elif (token == image_token_id) and (video_check_flg is True):
521
- input_token_type.append("video")
522
- else:
523
- input_token_type.append("text")
524
-
525
- input_type_group: list[tuple[str, int, int]] = []
526
- for key, group_iter in itertools.groupby(
527
- enumerate(input_token_type), lambda x: x[1]
528
- ):
529
- group_list = list(group_iter)
530
- start_index = group_list[0][0]
531
- end_index = group_list[-1][0] + 1
532
- input_type_group.append((key, start_index, end_index))
533
-
534
- video_frame_num = 1
535
- mm_data_idx = 0
536
- for modality_type, start_idx, end_idx in input_type_group:
537
- st_idx = (
538
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
539
- )
540
- if modality_type == "image":
541
- t, h, w = (
542
- image_grid_thw[mm_data_idx][0],
543
- image_grid_thw[mm_data_idx][1],
544
- image_grid_thw[mm_data_idx][2],
545
- )
546
- llm_grid_t, llm_grid_h, llm_grid_w = (
547
- t,
548
- h // spatial_merge_size,
549
- w // spatial_merge_size,
550
- )
551
-
552
- t_index = (
553
- torch.arange(llm_grid_t)
554
- .view(-1, 1)
555
- .expand(-1, llm_grid_h * llm_grid_w)
556
- .flatten()
557
- )
558
- h_index = (
559
- torch.arange(llm_grid_h)
560
- .view(1, -1, 1)
561
- .expand(llm_grid_t, -1, llm_grid_w)
562
- .flatten()
563
- )
564
- w_index = (
565
- torch.arange(llm_grid_w)
566
- .view(1, 1, -1)
567
- .expand(llm_grid_t, llm_grid_h, -1)
568
- .flatten()
569
- )
570
- llm_pos_ids_list.append(
571
- torch.stack([t_index, h_index, w_index]) + st_idx
572
- )
573
- mm_data_idx += 1
574
-
575
- elif modality_type == "video":
576
- t, h, w = (
577
- video_frame_num,
578
- image_grid_thw[mm_data_idx][1],
579
- image_grid_thw[mm_data_idx][2],
580
- )
581
- llm_grid_t, llm_grid_h, llm_grid_w = (
582
- t,
583
- h // spatial_merge_size,
584
- w // spatial_merge_size,
585
- )
586
-
587
- for t_idx in range(llm_grid_t):
588
- t_index = (
589
- torch.tensor(t_idx)
590
- .view(-1, 1)
591
- .expand(-1, llm_grid_h * llm_grid_w)
592
- .flatten()
593
- )
594
- h_index = (
595
- torch.arange(llm_grid_h)
596
- .view(1, -1, 1)
597
- .expand(1, -1, llm_grid_w)
598
- .flatten()
599
- )
600
- w_index = (
601
- torch.arange(llm_grid_w)
602
- .view(1, 1, -1)
603
- .expand(1, llm_grid_h, -1)
604
- .flatten()
605
- )
606
- llm_pos_ids_list.append(
607
- torch.stack([t_index, h_index, w_index]) + st_idx
608
- )
609
-
610
- mm_data_idx += 1
611
- video_frame_num += 1
612
-
613
- else:
614
- text_len = end_idx - start_idx
615
- llm_pos_ids_list.append(
616
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
617
- )
618
- video_frame_num = 1
619
-
620
- else:
621
- text_len = len(input_tokens)
622
- llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
623
-
624
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
625
- llm_positions = llm_positions[:, context_len:seq_len]
626
- mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
627
- return llm_positions, mrope_position_delta
628
-
629
- @classmethod
630
- def _qwen3vl_get_input_positions_tensor(
631
- cls,
632
- input_tokens: list[int],
633
- hf_config: PretrainedConfig,
634
- image_grid_thw: Union[list[list[int]], torch.Tensor],
635
- video_grid_thw: Union[list[list[int]], torch.Tensor],
636
- context_len: int = 0,
637
- seq_len: Optional[int] = None,
638
- ) -> tuple[torch.Tensor, int]:
639
- """Get mrope input positions and delta value."""
640
-
641
- video_grid_thw = [[1, h, w] for t, h, w in video_grid_thw for _ in range(t)]
642
-
643
- image_token_id = hf_config.image_token_id
644
- video_token_id = hf_config.video_token_id
645
- vision_start_token_id = hf_config.vision_start_token_id
646
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
647
-
648
- input_tokens_tensor = torch.tensor(input_tokens)
649
- vision_start_indices = torch.argwhere(
650
- input_tokens_tensor == vision_start_token_id
651
- ).squeeze(1)
652
- vision_tokens = input_tokens_tensor[vision_start_indices + 1]
653
- image_nums = (vision_tokens == image_token_id).sum()
654
- video_nums = (vision_tokens == video_token_id).sum()
655
- llm_pos_ids_list: list = []
656
-
657
- st = 0
658
- remain_images, remain_videos = image_nums, video_nums
659
-
660
- image_index, video_index = 0, 0
661
- for _ in range(image_nums + video_nums):
662
- if image_token_id in input_tokens and remain_images > 0:
663
- ed_image = input_tokens.index(image_token_id, st)
664
- else:
665
- ed_image = len(input_tokens) + 1
666
- if video_token_id in input_tokens and remain_videos > 0:
667
- ed_video = input_tokens.index(video_token_id, st)
668
- else:
669
- ed_video = len(input_tokens) + 1
670
- if ed_image < ed_video:
671
- t, h, w = (
672
- image_grid_thw[image_index][0],
673
- image_grid_thw[image_index][1],
674
- image_grid_thw[image_index][2],
675
- )
676
- image_index += 1
677
- remain_images -= 1
678
- ed = ed_image
679
- else:
680
- t, h, w = (
681
- video_grid_thw[video_index][0],
682
- video_grid_thw[video_index][1],
683
- video_grid_thw[video_index][2],
684
- )
685
- video_index += 1
686
- remain_videos -= 1
687
- ed = ed_video
688
-
689
- llm_grid_t, llm_grid_h, llm_grid_w = (
690
- t,
691
- h // spatial_merge_size,
692
- w // spatial_merge_size,
693
- )
694
- text_len = ed - st
695
-
696
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
697
- llm_pos_ids_list.append(
698
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
699
- )
700
-
701
- t_index = (
702
- torch.arange(llm_grid_t)
703
- .view(-1, 1)
704
- .expand(-1, llm_grid_h * llm_grid_w)
705
- .flatten()
706
- )
707
- h_index = (
708
- torch.arange(llm_grid_h)
709
- .view(1, -1, 1)
710
- .expand(llm_grid_t, -1, llm_grid_w)
711
- .flatten()
712
- )
713
- w_index = (
714
- torch.arange(llm_grid_w)
715
- .view(1, 1, -1)
716
- .expand(llm_grid_t, llm_grid_h, -1)
717
- .flatten()
718
- )
719
- llm_pos_ids_list.append(
720
- torch.stack([t_index, h_index, w_index]) + text_len + st_idx
721
- )
722
- st = ed + llm_grid_t * llm_grid_h * llm_grid_w
723
-
724
- if st < len(input_tokens):
725
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
726
- text_len = len(input_tokens) - st
727
- llm_pos_ids_list.append(
728
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
729
- )
730
-
731
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
732
- mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
733
- llm_positions = llm_positions[:, context_len:seq_len]
734
- return llm_positions, mrope_position_delta
735
-
736
- @classmethod
737
- def _ernie_get_input_positions_tensor(
738
- cls,
739
- input_tokens: list[int],
740
- hf_config: PretrainedConfig,
741
- image_grid_thw: Union[list[list[int]], torch.Tensor],
742
- video_grid_thw: Union[list[list[int]], torch.Tensor],
743
- context_len: int = 0,
744
- seq_len: Optional[int] = None,
745
- ) -> tuple[torch.Tensor, int]:
746
- """Get mrope input positions and delta value for Ernie VL."""
747
-
748
- image_token_id = hf_config.im_patch_id
749
- video_start_token_id = hf_config.video_start_token_id
750
- video_end_token_id = hf_config.video_end_token_id
751
- spatial_conv_size = hf_config.spatial_conv_size
752
- temporal_conv_size = hf_config.temporal_conv_size
753
- llm_pos_ids_list: list = []
754
-
755
- if not (image_grid_thw is None and video_grid_thw is None):
756
- if isinstance(image_grid_thw, torch.Tensor):
757
- image_grid_thw = image_grid_thw.tolist()
758
-
759
- input_token_type: list[str] = []
760
- video_check_flg = False
761
- for token in input_tokens:
762
- if token == video_start_token_id:
763
- video_check_flg = True
764
- elif token == video_end_token_id:
765
- video_check_flg = False
766
-
767
- if (token == image_token_id) and (video_check_flg is False):
768
- input_token_type.append("image")
769
- elif (token == image_token_id) and (video_check_flg is True):
770
- input_token_type.append("video")
771
- else:
772
- input_token_type.append("text")
773
-
774
- input_type_group: list[tuple[str, int, int]] = []
775
- for key, group_iter in itertools.groupby(
776
- enumerate(input_token_type), lambda x: x[1]
777
- ):
778
- group_list = list(group_iter)
779
- start_index = group_list[0][0]
780
- end_index = group_list[-1][0] + 1
781
- input_type_group.append((key, start_index, end_index))
782
-
783
- video_frame_num = 1
784
- mm_data_idx = 0
785
- for modality_type, start_idx, end_idx in input_type_group:
786
- st_idx = (
787
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
788
- )
789
- if modality_type == "image":
790
- t, h, w = (
791
- image_grid_thw[mm_data_idx][0],
792
- image_grid_thw[mm_data_idx][1],
793
- image_grid_thw[mm_data_idx][2],
794
- )
795
- llm_grid_t, llm_grid_h, llm_grid_w = (
796
- t,
797
- h // spatial_conv_size,
798
- w // spatial_conv_size,
799
- )
800
-
801
- t_index = (
802
- torch.arange(llm_grid_t)
803
- .view(-1, 1)
804
- .expand(-1, llm_grid_h * llm_grid_w)
805
- .flatten()
806
- )
807
- h_index = (
808
- torch.arange(llm_grid_h)
809
- .view(1, -1, 1)
810
- .expand(llm_grid_t, -1, llm_grid_w)
811
- .flatten()
812
- )
813
- w_index = (
814
- torch.arange(llm_grid_w)
815
- .view(1, 1, -1)
816
- .expand(llm_grid_t, llm_grid_h, -1)
817
- .flatten()
818
- )
819
- llm_pos_ids_list.append(
820
- torch.stack([t_index, h_index, w_index]) + st_idx
821
- )
822
- mm_data_idx += 1
823
-
824
- elif modality_type == "video":
825
- t, h, w = (
826
- video_grid_thw[mm_data_idx][0],
827
- video_grid_thw[mm_data_idx][1],
828
- video_grid_thw[mm_data_idx][2],
829
- )
830
- llm_grid_t, llm_grid_h, llm_grid_w = (
831
- t // temporal_conv_size,
832
- h // spatial_conv_size,
833
- w // spatial_conv_size,
834
- )
835
-
836
- for t_idx in range(llm_grid_t):
837
- t_index = (
838
- torch.tensor(t_idx)
839
- .view(-1, 1)
840
- .expand(-1, llm_grid_h * llm_grid_w)
841
- .flatten()
842
- )
843
- h_index = (
844
- torch.arange(llm_grid_h)
845
- .view(1, -1, 1)
846
- .expand(1, -1, llm_grid_w)
847
- .flatten()
848
- )
849
- w_index = (
850
- torch.arange(llm_grid_w)
851
- .view(1, 1, -1)
852
- .expand(1, llm_grid_h, -1)
853
- .flatten()
854
- )
855
- llm_pos_ids_list.append(
856
- torch.stack([t_index, h_index, w_index]) + st_idx
857
- )
858
-
859
- mm_data_idx += 1
860
- video_frame_num += 1
861
-
862
- else:
863
- text_len = end_idx - start_idx
864
- llm_pos_ids_list.append(
865
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
866
- )
867
- video_frame_num = 1
868
-
869
- else:
870
- text_len = len(input_tokens)
871
- llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
872
-
873
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
874
- llm_positions = llm_positions[:, context_len:seq_len]
875
- mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
876
- return llm_positions, mrope_position_delta
877
-
878
- @classmethod
879
- def _keye_get_input_positions_tensor(
880
- cls,
881
- input_tokens: list[int],
882
- hf_config: PretrainedConfig,
883
- image_grid_thw: Union[list[list[int]], torch.Tensor],
884
- video_grid_thw: Union[list[list[int]], torch.Tensor],
885
- context_len: int = 0,
886
- seq_len: Optional[int] = None,
887
- ) -> tuple[torch.Tensor, int]:
888
- if isinstance(video_grid_thw, list) and len(video_grid_thw) > 0:
889
- video_grid_thw = video_grid_thw[0]
890
- """Get mrope input positions and delta value (Keye series)."""
891
-
892
- def split_thw(grid_thw: Union[torch.Tensor, list[int]]) -> list[list[int]]:
893
- """
894
- Split grid_thw along the t dimension.
895
-
896
- Args:
897
- grid_thw: shape [N, 3] tensor or nested list of [t, h, w].
898
-
899
- Returns:
900
- List of [1, h, w] rows, repeated t times for each original row.
901
- """
902
-
903
- if isinstance(grid_thw, list):
904
- grid_thw = torch.tensor(grid_thw, dtype=torch.long)
905
-
906
- if grid_thw.numel() == 0:
907
- return []
908
-
909
- t, hw = grid_thw[:, 0], grid_thw[:, 1:]
910
- ones = torch.ones_like(hw[:, :1]) # [N,1]
911
- out = torch.cat([ones, hw], dim=1).repeat_interleave(t, dim=0)
912
- return out.tolist()
913
-
914
- video_grid_thw = split_thw(video_grid_thw)
915
-
916
- image_token_id = hf_config.image_token_id
917
- video_token_id = hf_config.video_token_id
918
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
919
-
920
- image_nums = len(image_grid_thw)
921
- frame_nums = len(video_grid_thw)
922
- llm_pos_ids_list: list = []
923
-
924
- st = 0
925
- remain_images, remain_frames = image_nums, frame_nums
926
-
927
- image_index, video_index = 0, 0
928
- for _ in range(image_nums + frame_nums):
929
- if remain_images > 0:
930
- try:
931
- ed_image = input_tokens.index(image_token_id, st)
932
- except ValueError:
933
- ed_image = len(input_tokens) + 1
934
- else:
935
- ed_image = len(input_tokens) + 1
936
- if remain_frames > 0:
937
- try:
938
- ed_video = input_tokens.index(video_token_id, st)
939
- except ValueError:
940
- ed_video = len(input_tokens) + 1
941
- else:
942
- ed_video = len(input_tokens) + 1
943
-
944
- if ed_image < ed_video:
945
- t, h, w = (
946
- image_grid_thw[image_index][0],
947
- image_grid_thw[image_index][1],
948
- image_grid_thw[image_index][2],
949
- )
950
- image_index += 1
951
- remain_images -= 1
952
- ed = ed_image
953
- else:
954
- t, h, w = (
955
- video_grid_thw[video_index][0],
956
- video_grid_thw[video_index][1],
957
- video_grid_thw[video_index][2],
958
- )
959
- video_index += 1
960
- remain_frames -= 1
961
- ed = ed_video
962
-
963
- llm_grid_t, llm_grid_h, llm_grid_w = (
964
- t,
965
- h // spatial_merge_size,
966
- w // spatial_merge_size,
967
- )
968
- text_len = ed - st
969
-
970
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
971
- llm_pos_ids_list.append(
972
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
973
- )
974
- t_index = (
975
- torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w)
976
- ).flatten()
977
- h_index = (
978
- torch.arange(llm_grid_h)
979
- .view(1, -1, 1)
980
- .expand(llm_grid_t, -1, llm_grid_w)
981
- .flatten()
982
- )
983
- w_index = (
984
- torch.arange(llm_grid_w)
985
- .view(1, 1, -1)
986
- .expand(llm_grid_t, llm_grid_h, -1)
987
- .flatten()
988
- )
989
- llm_pos_ids_list.append(
990
- torch.stack([t_index, h_index, w_index]) + text_len + st_idx
991
- )
992
- st = ed + llm_grid_t * llm_grid_h * llm_grid_w
993
-
994
- if st < len(input_tokens):
995
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
996
- text_len = len(input_tokens) - st
997
- llm_pos_ids_list.append(
998
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
999
- )
1000
-
1001
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1002
- mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1003
- llm_positions = llm_positions[:, context_len:seq_len]
1004
-
1005
- return llm_positions, mrope_position_delta
1006
-
1007
- @classmethod
1008
- def _vl_get_input_positions_tensor(
1009
- cls,
1010
- input_tokens: list[int],
1011
- hf_config: PretrainedConfig,
1012
- image_grid_thw: Union[list[list[int]], torch.Tensor],
1013
- video_grid_thw: Union[list[list[int]], torch.Tensor],
1014
- second_per_grid_ts: list[float],
1015
- context_len: int = 0,
1016
- seq_len: Optional[int] = None,
1017
- ) -> tuple[torch.Tensor, int]:
1018
- """Get mrope input positions and delta value."""
1019
-
1020
- image_token_id = hf_config.image_token_id
1021
- video_token_id = hf_config.video_token_id
1022
- vision_start_token_id = hf_config.vision_start_token_id
1023
- spatial_merge_size = hf_config.vision_config.spatial_merge_size
1024
- tokens_per_second = getattr(hf_config.vision_config, "tokens_per_second", 1.0)
1025
-
1026
- input_tokens_tensor = torch.tensor(input_tokens)
1027
- vision_start_indices = torch.argwhere(
1028
- input_tokens_tensor == vision_start_token_id
1029
- ).squeeze(1)
1030
- vision_tokens = input_tokens_tensor[vision_start_indices + 1]
1031
- image_nums = (vision_tokens == image_token_id).sum()
1032
- video_nums = (vision_tokens == video_token_id).sum()
1033
- llm_pos_ids_list: list = []
1034
-
1035
- st = 0
1036
- remain_images, remain_videos = image_nums, video_nums
1037
- image_index, video_index = 0, 0
1038
- for _ in range(image_nums + video_nums):
1039
- video_second_per_grid_t = 0.0
1040
- if remain_images > 0:
1041
- try:
1042
- ed_image = input_tokens.index(image_token_id, st)
1043
- except ValueError:
1044
- ed_image = len(input_tokens) + 1
1045
- else:
1046
- ed_image = len(input_tokens) + 1
1047
- if remain_videos > 0:
1048
- try:
1049
- ed_video = input_tokens.index(video_token_id, st)
1050
- except ValueError:
1051
- ed_video = len(input_tokens) + 1
1052
- else:
1053
- ed_video = len(input_tokens) + 1
1054
- if ed_image < ed_video:
1055
- t, h, w = (
1056
- image_grid_thw[image_index][0],
1057
- image_grid_thw[image_index][1],
1058
- image_grid_thw[image_index][2],
1059
- )
1060
- image_index += 1
1061
- remain_images -= 1
1062
- ed = ed_image
1063
- else:
1064
- t, h, w = (
1065
- video_grid_thw[video_index][0],
1066
- video_grid_thw[video_index][1],
1067
- video_grid_thw[video_index][2],
1068
- )
1069
- video_second_per_grid_t = 1.0
1070
- if second_per_grid_ts:
1071
- video_second_per_grid_t = second_per_grid_ts[video_index]
1072
- video_index += 1
1073
- remain_videos -= 1
1074
- ed = ed_video
1075
-
1076
- llm_grid_t, llm_grid_h, llm_grid_w = (
1077
- t,
1078
- h // spatial_merge_size,
1079
- w // spatial_merge_size,
1080
- )
1081
- text_len = ed - st
1082
-
1083
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1084
- llm_pos_ids_list.append(
1085
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1086
- )
1087
- t_index = (
1088
- torch.arange(llm_grid_t).view(-1, 1).expand(-1, llm_grid_h * llm_grid_w)
1089
- * video_second_per_grid_t
1090
- * tokens_per_second
1091
- ).flatten()
1092
- h_index = (
1093
- torch.arange(llm_grid_h)
1094
- .view(1, -1, 1)
1095
- .expand(llm_grid_t, -1, llm_grid_w)
1096
- .flatten()
1097
- )
1098
- w_index = (
1099
- torch.arange(llm_grid_w)
1100
- .view(1, 1, -1)
1101
- .expand(llm_grid_t, llm_grid_h, -1)
1102
- .flatten()
1103
- )
1104
- llm_pos_ids_list.append(
1105
- torch.stack([t_index, h_index, w_index]) + text_len + st_idx
1106
- )
1107
- st = ed + llm_grid_t * llm_grid_h * llm_grid_w
1108
-
1109
- if st < len(input_tokens):
1110
- st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1111
- text_len = len(input_tokens) - st
1112
- llm_pos_ids_list.append(
1113
- torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1114
- )
1115
-
1116
- llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1117
- mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1118
- llm_positions = llm_positions[:, context_len:seq_len]
1119
-
1120
- return llm_positions, mrope_position_delta
1121
-
1122
- @classmethod
1123
- def _omni_get_input_positions_tensor(
1124
- cls,
1125
- input_tokens: list[int],
1126
- hf_config: PretrainedConfig,
1127
- image_grid_thw: Union[list[list[int]], torch.Tensor],
1128
- video_grid_thw: Union[list[list[int]], torch.Tensor],
1129
- second_per_grid_ts: Optional[list[float]] = None,
1130
- context_len: int = 0,
1131
- seq_len: Optional[int] = None,
1132
- audio_feature_lengths: Optional[torch.Tensor] = None,
1133
- use_audio_in_video: bool = False,
1134
- ) -> tuple[torch.Tensor, int]:
1135
- """Get mrope input positions and delta value (Qwen2.5-Omni version).
1136
-
1137
- Differences from MRotaryEmbedding:
1138
- 1. Add audio support (and related `audio_feature_lengths`).
1139
- 2. Add `use_audio_in_video` option to read audio from video inputs.
1140
- In this case, audio and vision position ids will be split into
1141
- chunks and interleaved.
1142
-
1143
- Example:
1144
-
1145
- (V_i are vision position ids, A_i are audio position ids)
1146
-
1147
- |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
1148
- |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
1149
- """
1150
-
1151
- # TODO(fyabc): refactor and share more code with
1152
- # _vl_get_input_positions_tensor.
1153
- thinker_config = hf_config.thinker_config
1154
-
1155
- if isinstance(image_grid_thw, list):
1156
- image_grid_thw = torch.tensor(image_grid_thw)
1157
- if isinstance(video_grid_thw, list):
1158
- video_grid_thw = torch.tensor(video_grid_thw)
1159
-
1160
- audio_token_id = thinker_config.audio_token_index
1161
- image_token_id = thinker_config.image_token_index
1162
- video_token_id = thinker_config.video_token_index
1163
- audio_start_token_id = thinker_config.audio_start_token_id
1164
- audio_end_token_id = thinker_config.audio_end_token_id
1165
- vision_start_token_id = thinker_config.vision_start_token_id
1166
- vision_end_token_id = thinker_config.vision_end_token_id
1167
- seconds_per_chunk = thinker_config.seconds_per_chunk
1168
- spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1169
- tokens_per_second = getattr(
1170
- thinker_config.vision_config, "tokens_per_second", 25
1171
- )
1172
-
1173
- src_item = input_tokens
1174
- audio_seqlens = audio_feature_lengths
1175
- if not second_per_grid_ts:
1176
- second_per_grid_ts = [1] * video_grid_thw.shape[0]
1177
- audio_idx = 0
1178
- video_idx = 0
1179
- image_idx = 0
1180
- new_src_item: list[int] = []
1181
- llm_pos_ids_list: list[torch.Tensor] = []
1182
-
1183
- idx = 0
1184
- while idx < len(src_item):
1185
- new_src_item_len = len(new_src_item)
1186
- start_idx = (
1187
- llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1188
- )
1189
- if src_item[idx] not in [audio_token_id, video_token_id, image_token_id]:
1190
- if use_audio_in_video and idx > 0:
1191
- if (
1192
- src_item[idx] == vision_end_token_id
1193
- and src_item[idx - 1] == audio_end_token_id
1194
- ):
1195
- # processing the <|audio_eos|> before <|vision_eos|>
1196
- start_idx -= 1
1197
- elif (
1198
- src_item[idx] == audio_start_token_id
1199
- and src_item[idx - 1] == vision_start_token_id
1200
- ):
1201
- # processing the <|audio_bos|> after <|vision_eos|>
1202
- start_idx -= 1
1203
- new_src_item.append(src_item[idx])
1204
- llm_pos_ids = torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
1205
- llm_pos_ids_list.append(llm_pos_ids)
1206
- elif src_item[idx] == audio_token_id:
1207
- assert audio_seqlens is not None
1208
- audio_seqlen = audio_seqlens[audio_idx]
1209
- place_num = ((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1
1210
- new_src_item.extend([audio_token_id] * place_num)
1211
- llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx
1212
- llm_pos_ids_list.append(llm_pos_ids)
1213
- audio_idx += 1
1214
- elif src_item[idx] == image_token_id:
1215
- grid_t = image_grid_thw[image_idx][0]
1216
- grid_hs = image_grid_thw[:, 1]
1217
- grid_ws = image_grid_thw[:, 2]
1218
- t_index = torch.arange(grid_t) * 1 * tokens_per_second
1219
- llm_pos_ids = cls._get_llm_pos_ids_for_vision(
1220
- start_idx, image_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1221
- )
1222
- llm_pos_ids_list.append(llm_pos_ids)
1223
- vision_seqlen = image_grid_thw[image_idx].prod() // (
1224
- spatial_merge_size**2
1225
- )
1226
- new_src_item.extend([image_token_id] * vision_seqlen)
1227
- image_idx += 1
1228
- elif src_item[idx] == video_token_id and not use_audio_in_video:
1229
- grid_t = video_grid_thw[video_idx][0]
1230
- grid_hs = video_grid_thw[:, 1]
1231
- grid_ws = video_grid_thw[:, 2]
1232
- t_index = (
1233
- torch.arange(grid_t)
1234
- * second_per_grid_ts[video_idx]
1235
- * tokens_per_second
1236
- )
1237
- llm_pos_ids = cls._get_llm_pos_ids_for_vision(
1238
- start_idx, video_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1239
- )
1240
- llm_pos_ids_list.append(llm_pos_ids)
1241
- vision_seqlen = video_grid_thw[video_idx].prod() // (
1242
- spatial_merge_size**2
1243
- )
1244
- new_src_item.extend([video_token_id] * vision_seqlen)
1245
- video_idx += 1
1246
- else:
1247
- # read audio from video
1248
- assert audio_seqlens is not None
1249
- audio_seqlen = audio_seqlens[audio_idx]
1250
- vision_seqlen = video_grid_thw[video_idx].prod() // (
1251
- spatial_merge_size**2
1252
- )
1253
- grid_t = video_grid_thw[video_idx][0]
1254
- grid_h = video_grid_thw[video_idx][1]
1255
- grid_w = video_grid_thw[video_idx][2]
1256
- grid_hs = video_grid_thw[:, 1]
1257
- grid_ws = video_grid_thw[:, 2]
1258
- t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1259
- t_index = (
1260
- torch.arange(grid_t)
1261
- * second_per_grid_ts[video_idx]
1262
- * tokens_per_second
1263
- )
1264
- t_index_split_chunk = cls._split_list_into_ranges(
1265
- t_index, t_ntoken_per_chunk
1266
- )
1267
- place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
1268
- pure_audio_len = place_num - 2
1269
- added_audio_len = 0
1270
- audio_llm_pos_ids_list: list[torch.Tensor] = []
1271
- for t_chunk in t_index_split_chunk:
1272
- vision_ntoken_per_chunk = (
1273
- len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
1274
- )
1275
- new_src_item.extend([video_token_id] * vision_ntoken_per_chunk)
1276
- vision_llm_pos_ids_list = cls._get_llm_pos_ids_for_vision(
1277
- start_idx,
1278
- video_idx,
1279
- spatial_merge_size,
1280
- t_chunk,
1281
- grid_hs,
1282
- grid_ws,
1283
- ).split(1, dim=1)
1284
- llm_pos_ids_list.extend(vision_llm_pos_ids_list)
1285
- new_src_item.extend(
1286
- min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
1287
- * [audio_token_id]
1288
- )
1289
- audio_start_idx = (
1290
- start_idx
1291
- if len(audio_llm_pos_ids_list) == 0
1292
- else audio_llm_pos_ids_list[-1][0].item() + 1
1293
- )
1294
- if min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) > 0:
1295
- audio_llm_pos_ids_list = (
1296
- torch.arange(
1297
- min(
1298
- t_ntoken_per_chunk, pure_audio_len - added_audio_len
1299
- )
1300
- ).expand(3, -1)
1301
- + audio_start_idx
1302
- ).split(1, dim=1)
1303
- else:
1304
- audio_llm_pos_ids_list = []
1305
- added_audio_len += min(
1306
- t_ntoken_per_chunk, pure_audio_len - added_audio_len
1307
- )
1308
- llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1309
- if added_audio_len < pure_audio_len:
1310
- new_src_item.extend(
1311
- (pure_audio_len - added_audio_len) * [audio_token_id]
1312
- )
1313
- audio_llm_pos_ids_list = (
1314
- torch.arange(pure_audio_len - added_audio_len).expand(3, -1)
1315
- + llm_pos_ids_list[-1].max()
1316
- + 1
1317
- ).split(1, dim=1)
1318
- llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1319
- audio_idx += 1
1320
- video_idx += 1
1321
- # move to the next token
1322
- idx += len(new_src_item) - new_src_item_len
1323
-
1324
- llm_positions = torch.cat(llm_pos_ids_list, dim=1)
1325
- mrope_position_delta = (
1326
- torch.cat(llm_pos_ids_list, dim=1).max() + 1 - len(src_item)
1327
- )
1328
- llm_positions = llm_positions[:, context_len:seq_len]
1329
-
1330
- return llm_positions, mrope_position_delta
1331
-
1332
- @staticmethod
1333
- def _get_llm_pos_ids_for_vision(
1334
- start_idx: int,
1335
- vision_idx: int,
1336
- spatial_merge_size: int,
1337
- t_index: list[int],
1338
- grid_hs: torch.Tensor,
1339
- grid_ws: torch.Tensor,
1340
- ) -> torch.Tensor:
1341
- llm_pos_ids_list = []
1342
- llm_grid_h = grid_hs[vision_idx] // spatial_merge_size
1343
- llm_grid_w = grid_ws[vision_idx] // spatial_merge_size
1344
- h_index = (
1345
- torch.arange(llm_grid_h)
1346
- .view(1, -1, 1)
1347
- .expand(len(t_index), -1, llm_grid_w)
1348
- .flatten()
1349
- )
1350
- w_index = (
1351
- torch.arange(llm_grid_w)
1352
- .view(1, 1, -1)
1353
- .expand(len(t_index), llm_grid_h, -1)
1354
- .flatten()
1355
- )
1356
- t_index_tensor = (
1357
- torch.Tensor(t_index)
1358
- .to(llm_grid_h.device)
1359
- .view(-1, 1)
1360
- .expand(-1, llm_grid_h * llm_grid_w)
1361
- .long()
1362
- .flatten()
1363
- )
1364
- _llm_pos_ids = torch.stack([t_index_tensor, h_index, w_index])
1365
- llm_pos_ids_list.append(_llm_pos_ids + start_idx)
1366
- llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1)
1367
- return llm_pos_ids
1368
-
1369
- @staticmethod
1370
- def _split_list_into_ranges(lst: torch.Tensor, interval: int) -> list[list[int]]:
1371
- ranges: list[list[int]] = [[] for _ in range((max(lst) // interval) + 1)]
1372
- for num in lst:
1373
- index = num // interval
1374
- ranges[index].append(num)
1375
- return ranges
1376
-
413
@staticmethod
414
def get_next_input_positions(
415
mrope_position_delta: int,
@@ -1403,54 +439,3 @@ class MRotaryEmbedding(RotaryEmbedding):
439
dtype=out.dtype,
440
)
441
out[:, out_offset : out_offset + num_new_tokens] = values
1406
-
1407
- @classmethod
1408
- def omni_get_updates_use_audio_in_video(
1409
- cls,
1410
- thinker_config: PretrainedConfig,
1411
- audio_len: int,
1412
- video_grid_thw: Union[list[int], torch.Tensor],
1413
- video_second_per_grid_t: float,
1414
- ) -> list[int]:
1415
- """Get video prompt updates when `use_audio_in_video` is True.
1416
-
1417
- In this case, audio and vision update ids will be split into
1418
- chunks and interleaved (details in `_omni_get_input_positions_tensor`).
1419
-
1420
- <|video_bos|><|VIDEO|><|video_eos|> =>
1421
- <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>
1422
- """
1423
-
1424
- audio_token_id = thinker_config.audio_token_index
1425
- video_token_id = thinker_config.video_token_index
1426
- audio_start_token_id = thinker_config.audio_start_token_id
1427
- audio_end_token_id = thinker_config.audio_end_token_id
1428
- seconds_per_chunk = thinker_config.seconds_per_chunk
1429
- spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1430
- tokens_per_second = getattr(
1431
- thinker_config.vision_config, "tokens_per_second", 25
1432
- )
1433
-
1434
- grid_t = video_grid_thw[0]
1435
- grid_h = video_grid_thw[1]
1436
- grid_w = video_grid_thw[2]
1437
- t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1438
- t_index = torch.arange(grid_t) * video_second_per_grid_t * tokens_per_second
1439
- t_index_split_chunk = cls._split_list_into_ranges(t_index, t_ntoken_per_chunk)
1440
-
1441
- updates = [audio_start_token_id]
1442
- added_audio_len = 0
1443
- for t_chunk in t_index_split_chunk:
1444
- vision_ntoken_per_chunk = (
1445
- len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
1446
- )
1447
- updates.extend([video_token_id] * vision_ntoken_per_chunk)
1448
-
1449
- audio_chunk_size = min(t_ntoken_per_chunk, audio_len - added_audio_len)
1450
- updates.extend(audio_chunk_size * [audio_token_id])
1451
- added_audio_len += audio_chunk_size
1452
- if added_audio_len < audio_len:
1453
- updates.extend((audio_len - added_audio_len) * [audio_token_id])
1454
- updates.extend([audio_end_token_id])
1455
-
1456
- return updates
vllm/model_executor/models/ernie45_vl.py
+149
-2
@@ -23,6 +23,7 @@
23
# limitations under the License.
24
"""Inference-only Erine VL model compatible with HuggingFace weights."""
25
26
+import itertools
27
import math
28
from collections.abc import Iterable, Mapping, Sequence
29
from functools import partial
@@ -33,7 +34,7 @@ import torch
34
import torch.nn as nn
35
import torch.nn.functional as F
36
from einops import rearrange, repeat
36
-from transformers import BatchFeature
37
+from transformers import BatchFeature, PretrainedConfig
38
39
from vllm.attention.backends.registry import _Backend
40
from vllm.attention.layer import (
@@ -76,6 +77,7 @@ from .ernie45_vl_moe import Ernie4_5_VLMoeForCausalLM
77
from .interfaces import (
78
MultiModalEmbeddings,
79
SupportsLoRA,
80
+ SupportsMRoPE,
81
SupportsMultiModal,
82
SupportsPP,
83
)
@@ -1271,7 +1273,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1273
dummy_inputs=Ernie4_5_VLDummyInputsBuilder,
1274
)
1275
class Ernie4_5_VLMoeForConditionalGeneration(
1274
- nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
1276
+ nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1277
):
1278
merge_by_field_config = True
1279
@@ -1388,6 +1390,151 @@ class Ernie4_5_VLMoeForConditionalGeneration(
1390
else:
1391
self.visual_token_mask = None
1392
1393
+ @classmethod
1394
+ def get_mrope_input_positions(
1395
+ cls,
1396
+ input_tokens: list[int],
1397
+ hf_config: PretrainedConfig,
1398
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
1399
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
1400
+ context_len: int = 0,
1401
+ seq_len: Optional[int] = None,
1402
+ second_per_grid_ts: Optional[list[float]] = None,
1403
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1404
+ use_audio_in_video: bool = False,
1405
+ ) -> tuple[torch.Tensor, int]:
1406
+ """Get mrope input positions and delta value for Ernie VL."""
1407
+
1408
+ image_token_id = hf_config.im_patch_id
1409
+ video_start_token_id = hf_config.video_start_token_id
1410
+ video_end_token_id = hf_config.video_end_token_id
1411
+ spatial_conv_size = hf_config.spatial_conv_size
1412
+ temporal_conv_size = hf_config.temporal_conv_size
1413
+ llm_pos_ids_list: list = []
1414
+
1415
+ if not (image_grid_thw is None and video_grid_thw is None):
1416
+ if isinstance(image_grid_thw, torch.Tensor):
1417
+ image_grid_thw = image_grid_thw.tolist()
1418
+
1419
+ input_token_type: list[str] = []
1420
+ video_check_flg = False
1421
+ for token in input_tokens:
1422
+ if token == video_start_token_id:
1423
+ video_check_flg = True
1424
+ elif token == video_end_token_id:
1425
+ video_check_flg = False
1426
+
1427
+ if (token == image_token_id) and (video_check_flg is False):
1428
+ input_token_type.append("image")
1429
+ elif (token == image_token_id) and (video_check_flg is True):
1430
+ input_token_type.append("video")
1431
+ else:
1432
+ input_token_type.append("text")
1433
+
1434
+ input_type_group: list[tuple[str, int, int]] = []
1435
+ for key, group_iter in itertools.groupby(
1436
+ enumerate(input_token_type), lambda x: x[1]
1437
+ ):
1438
+ group_list = list(group_iter)
1439
+ start_index = group_list[0][0]
1440
+ end_index = group_list[-1][0] + 1
1441
+ input_type_group.append((key, start_index, end_index))
1442
+
1443
+ video_frame_num = 1
1444
+ mm_data_idx = 0
1445
+ for modality_type, start_idx, end_idx in input_type_group:
1446
+ st_idx = (
1447
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1448
+ )
1449
+ if modality_type == "image":
1450
+ t, h, w = (
1451
+ image_grid_thw[mm_data_idx][0],
1452
+ image_grid_thw[mm_data_idx][1],
1453
+ image_grid_thw[mm_data_idx][2],
1454
+ )
1455
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1456
+ t,
1457
+ h // spatial_conv_size,
1458
+ w // spatial_conv_size,
1459
+ )
1460
+
1461
+ t_index = (
1462
+ torch.arange(llm_grid_t)
1463
+ .view(-1, 1)
1464
+ .expand(-1, llm_grid_h * llm_grid_w)
1465
+ .flatten()
1466
+ )
1467
+ h_index = (
1468
+ torch.arange(llm_grid_h)
1469
+ .view(1, -1, 1)
1470
+ .expand(llm_grid_t, -1, llm_grid_w)
1471
+ .flatten()
1472
+ )
1473
+ w_index = (
1474
+ torch.arange(llm_grid_w)
1475
+ .view(1, 1, -1)
1476
+ .expand(llm_grid_t, llm_grid_h, -1)
1477
+ .flatten()
1478
+ )
1479
+ llm_pos_ids_list.append(
1480
+ torch.stack([t_index, h_index, w_index]) + st_idx
1481
+ )
1482
+ mm_data_idx += 1
1483
+
1484
+ elif modality_type == "video":
1485
+ t, h, w = (
1486
+ video_grid_thw[mm_data_idx][0],
1487
+ video_grid_thw[mm_data_idx][1],
1488
+ video_grid_thw[mm_data_idx][2],
1489
+ )
1490
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1491
+ t // temporal_conv_size,
1492
+ h // spatial_conv_size,
1493
+ w // spatial_conv_size,
1494
+ )
1495
+
1496
+ for t_idx in range(llm_grid_t):
1497
+ t_index = (
1498
+ torch.tensor(t_idx)
1499
+ .view(-1, 1)
1500
+ .expand(-1, llm_grid_h * llm_grid_w)
1501
+ .flatten()
1502
+ )
1503
+ h_index = (
1504
+ torch.arange(llm_grid_h)
1505
+ .view(1, -1, 1)
1506
+ .expand(1, -1, llm_grid_w)
1507
+ .flatten()
1508
+ )
1509
+ w_index = (
1510
+ torch.arange(llm_grid_w)
1511
+ .view(1, 1, -1)
1512
+ .expand(1, llm_grid_h, -1)
1513
+ .flatten()
1514
+ )
1515
+ llm_pos_ids_list.append(
1516
+ torch.stack([t_index, h_index, w_index]) + st_idx
1517
+ )
1518
+
1519
+ mm_data_idx += 1
1520
+ video_frame_num += 1
1521
+
1522
+ else:
1523
+ text_len = end_idx - start_idx
1524
+ llm_pos_ids_list.append(
1525
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1526
+ )
1527
+ video_frame_num = 1
1528
+
1529
+ else:
1530
+ text_len = len(input_tokens)
1531
+ llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
1532
+
1533
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1534
+ llm_positions = llm_positions[:, context_len:seq_len]
1535
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1536
+ return llm_positions, mrope_position_delta
1537
+
1538
def get_language_model(self) -> torch.nn.Module:
1539
return self.language_model
1540
vllm/model_executor/models/glm4v.py
+150
-2
@@ -5,6 +5,7 @@
5
# https://github.com/zai-org/CogAgent
6
"""Inference-only CogAgent model compatible with THUDM weights."""
7
8
+import itertools
9
from argparse import Namespace
10
from collections.abc import Mapping, Sequence
11
from typing import Annotated, Literal, Optional, Union
@@ -14,7 +15,7 @@ from torch import nn
15
from torch.nn import LayerNorm
16
from torchvision import transforms
17
from torchvision.transforms import InterpolationMode
17
-from transformers import BatchFeature, PreTrainedTokenizer, TensorType
18
+from transformers import BatchFeature, PretrainedConfig, PreTrainedTokenizer, TensorType
19
from transformers.image_utils import ImageInput
20
from transformers.tokenization_utils_base import TextInput
21
@@ -54,6 +55,7 @@ from .chatglm import ChatGLMBaseModel, ChatGLMModel
55
from .interfaces import (
56
MultiModalEmbeddings,
57
SupportsLoRA,
58
+ SupportsMRoPE,
59
SupportsMultiModal,
60
SupportsPP,
61
)
@@ -554,7 +556,9 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
556
info=GLM4VProcessingInfo,
557
dummy_inputs=GLM4VDummyInputsBuilder,
558
)
557
-class GLM4VForCausalLM(ChatGLMBaseModel, SupportsMultiModal, SupportsLoRA, SupportsPP):
559
+class GLM4VForCausalLM(
560
+ ChatGLMBaseModel, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
561
+):
562
merge_by_field_config = True
563
564
packed_modules_mapping = {
@@ -615,6 +619,150 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsMultiModal, SupportsLoRA, Suppo
619
620
return self.transformer.vision(pixel_values)
621
622
+ @classmethod
623
+ def get_mrope_input_positions(
624
+ cls,
625
+ input_tokens: list[int],
626
+ hf_config: PretrainedConfig,
627
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
628
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
629
+ context_len: int = 0,
630
+ seq_len: Optional[int] = None,
631
+ second_per_grid_ts: Optional[list[float]] = None,
632
+ audio_feature_lengths: Optional[torch.Tensor] = None,
633
+ use_audio_in_video: bool = False,
634
+ ) -> tuple[torch.Tensor, int]:
635
+ """Get mrope input positions and delta value for GLM4V."""
636
+
637
+ image_token_id = hf_config.image_token_id
638
+ video_start_token_id = hf_config.video_start_token_id
639
+ video_end_token_id = hf_config.video_end_token_id
640
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
641
+ llm_pos_ids_list: list = []
642
+
643
+ if not (image_grid_thw is None and video_grid_thw is None):
644
+ if isinstance(image_grid_thw, torch.Tensor):
645
+ image_grid_thw = image_grid_thw.tolist()
646
+
647
+ input_token_type: list[str] = []
648
+ video_check_flg = False
649
+ for token in input_tokens:
650
+ if token == video_start_token_id:
651
+ video_check_flg = True
652
+ elif token == video_end_token_id:
653
+ video_check_flg = False
654
+
655
+ if (token == image_token_id) and (video_check_flg is False):
656
+ input_token_type.append("image")
657
+ elif (token == image_token_id) and (video_check_flg is True):
658
+ input_token_type.append("video")
659
+ else:
660
+ input_token_type.append("text")
661
+
662
+ input_type_group: list[tuple[str, int, int]] = []
663
+ for key, group_iter in itertools.groupby(
664
+ enumerate(input_token_type), lambda x: x[1]
665
+ ):
666
+ group_list = list(group_iter)
667
+ start_index = group_list[0][0]
668
+ end_index = group_list[-1][0] + 1
669
+ input_type_group.append((key, start_index, end_index))
670
+
671
+ video_frame_num = 1
672
+ mm_data_idx = 0
673
+ for modality_type, start_idx, end_idx in input_type_group:
674
+ st_idx = (
675
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
676
+ )
677
+ if modality_type == "image":
678
+ t, h, w = (
679
+ image_grid_thw[mm_data_idx][0],
680
+ image_grid_thw[mm_data_idx][1],
681
+ image_grid_thw[mm_data_idx][2],
682
+ )
683
+ llm_grid_t, llm_grid_h, llm_grid_w = (
684
+ t,
685
+ h // spatial_merge_size,
686
+ w // spatial_merge_size,
687
+ )
688
+
689
+ t_index = (
690
+ torch.arange(llm_grid_t)
691
+ .view(-1, 1)
692
+ .expand(-1, llm_grid_h * llm_grid_w)
693
+ .flatten()
694
+ )
695
+ h_index = (
696
+ torch.arange(llm_grid_h)
697
+ .view(1, -1, 1)
698
+ .expand(llm_grid_t, -1, llm_grid_w)
699
+ .flatten()
700
+ )
701
+ w_index = (
702
+ torch.arange(llm_grid_w)
703
+ .view(1, 1, -1)
704
+ .expand(llm_grid_t, llm_grid_h, -1)
705
+ .flatten()
706
+ )
707
+ llm_pos_ids_list.append(
708
+ torch.stack([t_index, h_index, w_index]) + st_idx
709
+ )
710
+ mm_data_idx += 1
711
+
712
+ elif modality_type == "video":
713
+ t, h, w = (
714
+ video_frame_num,
715
+ image_grid_thw[mm_data_idx][1],
716
+ image_grid_thw[mm_data_idx][2],
717
+ )
718
+ llm_grid_t, llm_grid_h, llm_grid_w = (
719
+ t,
720
+ h // spatial_merge_size,
721
+ w // spatial_merge_size,
722
+ )
723
+
724
+ for t_idx in range(llm_grid_t):
725
+ t_index = (
726
+ torch.tensor(t_idx)
727
+ .view(-1, 1)
728
+ .expand(-1, llm_grid_h * llm_grid_w)
729
+ .flatten()
730
+ )
731
+ h_index = (
732
+ torch.arange(llm_grid_h)
733
+ .view(1, -1, 1)
734
+ .expand(1, -1, llm_grid_w)
735
+ .flatten()
736
+ )
737
+ w_index = (
738
+ torch.arange(llm_grid_w)
739
+ .view(1, 1, -1)
740
+ .expand(1, llm_grid_h, -1)
741
+ .flatten()
742
+ )
743
+ llm_pos_ids_list.append(
744
+ torch.stack([t_index, h_index, w_index]) + st_idx
745
+ )
746
+
747
+ mm_data_idx += 1
748
+ video_frame_num += 1
749
+
750
+ else:
751
+ text_len = end_idx - start_idx
752
+ llm_pos_ids_list.append(
753
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
754
+ )
755
+ video_frame_num = 1
756
+
757
+ else:
758
+ text_len = len(input_tokens)
759
+ llm_pos_ids_list.append(torch.arange(text_len).view(1, -1).expand(3, -1))
760
+
761
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
762
+ llm_positions = llm_positions[:, context_len:seq_len]
763
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
764
+ return llm_positions, mrope_position_delta
765
+
766
def get_language_model(self) -> torch.nn.Module:
767
return self.transformer
768
vllm/model_executor/models/keye_vl1_5.py
+142
-2
@@ -38,7 +38,7 @@ from vllm.multimodal.processing import (
38
)
39
from vllm.utils.tensor_schema import TensorSchema, TensorShape
40
41
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
41
+from .interfaces import SupportsLoRA, SupportsMRoPE, SupportsMultiModal, SupportsPP
42
from .keye import (
43
BaseKeyeModule,
44
BaseMultiModalProcessor,
@@ -493,7 +493,7 @@ class KeyeVL1_5DummyInputsBuilder(
493
dummy_inputs=KeyeVL1_5DummyInputsBuilder,
494
)
495
class KeyeVL1_5ForConditionalGeneration(
496
- BaseKeyeModule, SupportsMultiModal, SupportsLoRA, SupportsPP
496
+ BaseKeyeModule, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
497
):
498
def _build_projector(
499
self,
@@ -589,3 +589,143 @@ class KeyeVL1_5ForConditionalGeneration(
589
end = patch_cu_seqlens[idx + 1]
590
new_video_embeds.append(video_embeds[start:end])
591
return tuple(new_video_embeds)
592
+
593
+ @classmethod
594
+ def get_mrope_input_positions(
595
+ cls,
596
+ input_tokens: list[int],
597
+ hf_config: PretrainedConfig,
598
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
599
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
600
+ context_len: int = 0,
601
+ seq_len: Optional[int] = None,
602
+ second_per_grid_ts: Optional[list[float]] = None,
603
+ audio_feature_lengths: Optional[torch.Tensor] = None,
604
+ use_audio_in_video: bool = False,
605
+ ) -> tuple[torch.Tensor, int]:
606
+ if isinstance(video_grid_thw, list) and len(video_grid_thw) > 0:
607
+ video_grid_thw = video_grid_thw[0]
608
+ """Get mrope input positions and delta value (Keye series)."""
609
+
610
+ def split_thw(grid_thw: Union[torch.Tensor, list[int]]) -> list[list[int]]:
611
+ """
612
+ Split grid_thw along the t dimension.
613
+
614
+ Args:
615
+ grid_thw: shape [N, 3] tensor or nested list of [t, h, w].
616
+
617
+ Returns:
618
+ List of [1, h, w] rows, repeated t times for each original row.
619
+ """
620
+
621
+ if isinstance(grid_thw, list):
622
+ grid_thw = torch.tensor(grid_thw, dtype=torch.long)
623
+
624
+ if grid_thw.numel() == 0:
625
+ return []
626
+
627
+ t, hw = grid_thw[:, 0], grid_thw[:, 1:]
628
+ ones = torch.ones_like(hw[:, :1]) # [N,1]
629
+ out = torch.cat([ones, hw], dim=1).repeat_interleave(t, dim=0)
630
+ return out.tolist()
631
+
632
+ video_grid_thw = split_thw(video_grid_thw)
633
+
634
+ image_token_id = hf_config.image_token_id
635
+ video_token_id = hf_config.video_token_id
636
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
637
+
638
+ image_nums = len(image_grid_thw)
639
+ frame_nums = len(video_grid_thw)
640
+ llm_pos_ids_list: list = []
641
+
642
+ st = 0
643
+ remain_images, remain_frames = image_nums, frame_nums
644
+
645
+ image_index, video_index = 0, 0
646
+ for _ in range(image_nums + frame_nums):
647
+ if remain_images > 0:
648
+ try:
649
+ ed_image = input_tokens.index(image_token_id, st)
650
+ except ValueError:
651
+ ed_image = len(input_tokens) + 1
652
+ else:
653
+ ed_image = len(input_tokens) + 1
654
+ if remain_frames > 0:
655
+ try:
656
+ ed_video = input_tokens.index(video_token_id, st)
657
+ except ValueError:
658
+ ed_video = len(input_tokens) + 1
659
+ else:
660
+ ed_video = len(input_tokens) + 1
661
+
662
+ if ed_image < ed_video:
663
+ t, h, w = (
664
+ image_grid_thw[image_index][0],
665
+ image_grid_thw[image_index][1],
666
+ image_grid_thw[image_index][2],
667
+ )
668
+ image_index += 1
669
+ remain_images -= 1
670
+ ed = ed_image
671
+ else:
672
+ t, h, w = (
673
+ video_grid_thw[video_index][0],
674
+ video_grid_thw[video_index][1],
675
+ video_grid_thw[video_index][2],
676
+ )
677
+ video_index += 1
678
+ remain_frames -= 1
679
+ ed = ed_video
680
+
681
+ llm_grid_t, llm_grid_h, llm_grid_w = (
682
+ t,
683
+ h // spatial_merge_size,
684
+ w // spatial_merge_size,
685
+ )
686
+ text_len = ed - st
687
+
688
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
689
+ llm_pos_ids_list.append(
690
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
691
+ )
692
+
693
+ t_index = (
694
+ (
695
+ torch.arange(llm_grid_t)
696
+ .view(-1, 1)
697
+ .expand(-1, llm_grid_h * llm_grid_w)
698
+ )
699
+ .long()
700
+ .flatten()
701
+ )
702
+
703
+ h_index = (
704
+ torch.arange(llm_grid_h)
705
+ .view(1, -1, 1)
706
+ .expand(llm_grid_t, -1, llm_grid_w)
707
+ .flatten()
708
+ )
709
+ w_index = (
710
+ torch.arange(llm_grid_w)
711
+ .view(1, 1, -1)
712
+ .expand(llm_grid_t, llm_grid_h, -1)
713
+ .flatten()
714
+ )
715
+ llm_pos_ids_list.append(
716
+ torch.stack([t_index, h_index, w_index]) + text_len + st_idx
717
+ )
718
+ st = ed + llm_grid_t * llm_grid_h * llm_grid_w
719
+
720
+ if st < len(input_tokens):
721
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
722
+ text_len = len(input_tokens) - st
723
+ llm_pos_ids_list.append(
724
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
725
+ )
726
+
727
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
728
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
729
+ llm_positions = llm_positions[:, context_len:seq_len]
730
+
731
+ return llm_positions, mrope_position_delta
vllm/model_executor/models/qwen2_5_omni_thinker.py
+269
-2
@@ -29,6 +29,7 @@ from typing import Annotated, Any, Callable, Literal, Optional, Union
29
30
import torch
31
import torch.nn as nn
32
+from transformers import PretrainedConfig
33
from transformers.feature_extraction_utils import BatchFeature
34
from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
35
Qwen2_5OmniConfig,
@@ -45,7 +46,6 @@ from transformers.models.whisper import WhisperFeatureExtractor
46
from vllm.config import VllmConfig
47
from vllm.config.multimodal import BaseDummyOptions
48
from vllm.logger import init_logger
48
-from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
49
from vllm.model_executor.models.module_mapping import MultiModelKeys
50
from vllm.model_executor.models.qwen2_5_vl import (
51
Qwen2_5_VisionTransformer,
@@ -93,6 +93,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
93
from .interfaces import (
94
MultiModalEmbeddings,
95
SupportsLoRA,
96
+ SupportsMRoPE,
97
SupportsMultiModal,
98
SupportsPP,
99
)
@@ -101,7 +102,9 @@ from .utils import (
102
WeightsMapper,
103
init_vllm_registered_model,
104
maybe_prefix,
105
+ split_list_into_ranges,
106
)
107
+from .vision import get_llm_pos_ids_for_vision
108
109
try:
110
import flash_attn
@@ -412,6 +415,59 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
415
416
return prompt_ids, mm_placeholders
417
418
+ @classmethod
419
+ def omni_get_updates_use_audio_in_video(
420
+ cls,
421
+ thinker_config: PretrainedConfig,
422
+ audio_len: int,
423
+ video_grid_thw: Union[list[int], torch.Tensor],
424
+ video_second_per_grid_t: float,
425
+ ) -> list[int]:
426
+ """Get video prompt updates when `use_audio_in_video` is True.
427
+
428
+ In this case, audio and vision update ids will be split into
429
+ chunks and interleaved (details in `_omni_get_input_positions_tensor`).
430
+
431
+ <|video_bos|><|VIDEO|><|video_eos|> =>
432
+ <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>
433
+ """
434
+
435
+ audio_token_id = thinker_config.audio_token_index
436
+ video_token_id = thinker_config.video_token_index
437
+ audio_start_token_id = thinker_config.audio_start_token_id
438
+ audio_end_token_id = thinker_config.audio_end_token_id
439
+ seconds_per_chunk = thinker_config.seconds_per_chunk
440
+ spatial_merge_size = thinker_config.vision_config.spatial_merge_size
441
+ tokens_per_second = getattr(
442
+ thinker_config.vision_config, "tokens_per_second", 25
443
+ )
444
+
445
+ grid_t = video_grid_thw[0]
446
+ grid_h = video_grid_thw[1]
447
+ grid_w = video_grid_thw[2]
448
+ t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
449
+ t_index = (
450
+ torch.arange(grid_t) * video_second_per_grid_t * tokens_per_second
451
+ ).long()
452
+ t_index_split_chunk = split_list_into_ranges(t_index, t_ntoken_per_chunk)
453
+
454
+ updates = [audio_start_token_id]
455
+ added_audio_len = 0
456
+ for t_chunk in t_index_split_chunk:
457
+ vision_ntoken_per_chunk = (
458
+ len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
459
+ )
460
+ updates.extend([video_token_id] * vision_ntoken_per_chunk)
461
+
462
+ audio_chunk_size = min(t_ntoken_per_chunk, audio_len - added_audio_len)
463
+ updates.extend(audio_chunk_size * [audio_token_id])
464
+ added_audio_len += audio_chunk_size
465
+ if added_audio_len < audio_len:
466
+ updates.extend((audio_len - added_audio_len) * [audio_token_id])
467
+ updates.extend([audio_end_token_id])
468
+
469
+ return updates
470
+
471
def _get_prompt_updates(
472
self,
473
mm_items: MultiModalDataItems,
@@ -491,7 +547,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
547
else:
548
video_second_per_grid_t = 1.0
549
494
- return MRotaryEmbedding.omni_get_updates_use_audio_in_video(
550
+ return self.omni_get_updates_use_audio_in_video(
551
thinker_config=thinker_config,
552
audio_len=audio_num_features,
553
video_grid_thw=video_grid_thw,
@@ -808,6 +864,7 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
864
SupportsMultiModal,
865
SupportsPP,
866
SupportsLoRA,
867
+ SupportsMRoPE,
868
Qwen2_5OmniConditionalGenerationMixin,
869
):
870
hf_to_vllm_mapper = WeightsMapper(
@@ -929,6 +986,216 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
986
def get_language_model(self) -> torch.nn.Module:
987
return self.language_model
988
989
+ @classmethod
990
+ def get_mrope_input_positions(
991
+ cls,
992
+ input_tokens: list[int],
993
+ hf_config: PretrainedConfig,
994
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
995
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
996
+ second_per_grid_ts: Optional[list[float]] = None,
997
+ context_len: int = 0,
998
+ seq_len: Optional[int] = None,
999
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1000
+ use_audio_in_video: bool = False,
1001
+ ) -> tuple[torch.Tensor, int]:
1002
+ """Get mrope input positions and delta value (Qwen2.5-Omni version).
1003
+
1004
+ Differences from MRotaryEmbedding:
1005
+ 1. Add audio support (and related `audio_feature_lengths`).
1006
+ 2. Add `use_audio_in_video` option to read audio from video inputs.
1007
+ In this case, audio and vision position ids will be split into
1008
+ chunks and interleaved.
1009
+
1010
+ Example:
1011
+
1012
+ (V_i are vision position ids, A_i are audio position ids)
1013
+
1014
+ |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
1015
+ |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
1016
+ """
1017
+
1018
+ # TODO(fyabc): refactor and share more code with
1019
+ # _vl_get_input_positions_tensor.
1020
+
1021
+ thinker_config = hf_config.thinker_config
1022
+ audio_token_id = thinker_config.audio_token_index
1023
+ image_token_id = thinker_config.image_token_index
1024
+ video_token_id = thinker_config.video_token_index
1025
+ audio_start_token_id = thinker_config.audio_start_token_id
1026
+ audio_end_token_id = thinker_config.audio_end_token_id
1027
+ vision_start_token_id = thinker_config.vision_start_token_id
1028
+ vision_end_token_id = thinker_config.vision_end_token_id
1029
+ seconds_per_chunk = thinker_config.seconds_per_chunk
1030
+ spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1031
+ tokens_per_second = getattr(
1032
+ thinker_config.vision_config, "tokens_per_second", 25
1033
+ )
1034
+
1035
+ if isinstance(image_grid_thw, list):
1036
+ image_grid_thw = torch.tensor(image_grid_thw)
1037
+ if isinstance(video_grid_thw, list):
1038
+ video_grid_thw = torch.tensor(video_grid_thw)
1039
+
1040
+ src_item = input_tokens
1041
+ audio_seqlens = audio_feature_lengths
1042
+ if not second_per_grid_ts:
1043
+ second_per_grid_ts = [1] * video_grid_thw.shape[0]
1044
+ audio_idx = 0
1045
+ video_idx = 0
1046
+ image_idx = 0
1047
+ new_src_item: list[int] = []
1048
+ llm_pos_ids_list: list[torch.Tensor] = []
1049
+
1050
+ idx = 0
1051
+ while idx < len(src_item):
1052
+ new_src_item_len = len(new_src_item)
1053
+ start_idx = (
1054
+ llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1055
+ )
1056
+ if src_item[idx] not in [audio_token_id, video_token_id, image_token_id]:
1057
+ if use_audio_in_video and idx > 0:
1058
+ if (
1059
+ src_item[idx] == vision_end_token_id
1060
+ and src_item[idx - 1] == audio_end_token_id
1061
+ ):
1062
+ # processing the <|audio_eos|> before <|vision_eos|>
1063
+ start_idx -= 1
1064
+ elif (
1065
+ src_item[idx] == audio_start_token_id
1066
+ and src_item[idx - 1] == vision_start_token_id
1067
+ ):
1068
+ # processing the <|audio_bos|> after <|vision_eos|>
1069
+ start_idx -= 1
1070
+ new_src_item.append(src_item[idx])
1071
+ llm_pos_ids = torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
1072
+ llm_pos_ids_list.append(llm_pos_ids)
1073
+ elif src_item[idx] == audio_token_id:
1074
+ assert audio_seqlens is not None
1075
+ audio_seqlen = audio_seqlens[audio_idx]
1076
+ place_num = ((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1
1077
+ new_src_item.extend([audio_token_id] * place_num)
1078
+ llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx
1079
+ llm_pos_ids_list.append(llm_pos_ids)
1080
+ audio_idx += 1
1081
+ elif src_item[idx] == image_token_id:
1082
+ grid_t = image_grid_thw[image_idx][0]
1083
+ grid_hs = image_grid_thw[:, 1]
1084
+ grid_ws = image_grid_thw[:, 2]
1085
+ t_index = (torch.arange(grid_t) * 1 * tokens_per_second).long()
1086
+ llm_pos_ids = get_llm_pos_ids_for_vision(
1087
+ start_idx, image_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1088
+ )
1089
+ llm_pos_ids_list.append(llm_pos_ids)
1090
+ vision_seqlen = image_grid_thw[image_idx].prod() // (
1091
+ spatial_merge_size**2
1092
+ )
1093
+ new_src_item.extend([image_token_id] * vision_seqlen)
1094
+ image_idx += 1
1095
+ elif src_item[idx] == video_token_id and not use_audio_in_video:
1096
+ grid_t = video_grid_thw[video_idx][0]
1097
+ grid_hs = video_grid_thw[:, 1]
1098
+ grid_ws = video_grid_thw[:, 2]
1099
+ t_index = (
1100
+ torch.arange(grid_t)
1101
+ * second_per_grid_ts[video_idx]
1102
+ * tokens_per_second
1103
+ ).long()
1104
+ llm_pos_ids = get_llm_pos_ids_for_vision(
1105
+ start_idx, video_idx, spatial_merge_size, t_index, grid_hs, grid_ws
1106
+ )
1107
+ llm_pos_ids_list.append(llm_pos_ids)
1108
+ vision_seqlen = video_grid_thw[video_idx].prod() // (
1109
+ spatial_merge_size**2
1110
+ )
1111
+ new_src_item.extend([video_token_id] * vision_seqlen)
1112
+ video_idx += 1
1113
+ else:
1114
+ # read audio from video
1115
+ assert audio_seqlens is not None
1116
+ audio_seqlen = audio_seqlens[audio_idx]
1117
+ vision_seqlen = video_grid_thw[video_idx].prod() // (
1118
+ spatial_merge_size**2
1119
+ )
1120
+ grid_t = video_grid_thw[video_idx][0]
1121
+ grid_h = video_grid_thw[video_idx][1]
1122
+ grid_w = video_grid_thw[video_idx][2]
1123
+ grid_hs = video_grid_thw[:, 1]
1124
+ grid_ws = video_grid_thw[:, 2]
1125
+ t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1126
+ t_index = (
1127
+ torch.arange(grid_t)
1128
+ * second_per_grid_ts[video_idx]
1129
+ * tokens_per_second
1130
+ ).long()
1131
+ t_index_split_chunk = split_list_into_ranges(
1132
+ t_index, t_ntoken_per_chunk
1133
+ )
1134
+ place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
1135
+ pure_audio_len = place_num - 2
1136
+ added_audio_len = 0
1137
+ audio_llm_pos_ids_list: list[torch.Tensor] = []
1138
+ for t_chunk in t_index_split_chunk:
1139
+ vision_ntoken_per_chunk = (
1140
+ len(t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
1141
+ )
1142
+ new_src_item.extend([video_token_id] * vision_ntoken_per_chunk)
1143
+ vision_llm_pos_ids_list = get_llm_pos_ids_for_vision(
1144
+ start_idx,
1145
+ video_idx,
1146
+ spatial_merge_size,
1147
+ t_chunk,
1148
+ grid_hs,
1149
+ grid_ws,
1150
+ ).split(1, dim=1)
1151
+ llm_pos_ids_list.extend(vision_llm_pos_ids_list)
1152
+ new_src_item.extend(
1153
+ min(t_ntoken_per_chunk, pure_audio_len - added_audio_len)
1154
+ * [audio_token_id]
1155
+ )
1156
+ audio_start_idx = (
1157
+ start_idx
1158
+ if len(audio_llm_pos_ids_list) == 0
1159
+ else audio_llm_pos_ids_list[-1][0].item() + 1
1160
+ )
1161
+ if min(t_ntoken_per_chunk, pure_audio_len - added_audio_len) > 0:
1162
+ audio_llm_pos_ids_list = (
1163
+ torch.arange(
1164
+ min(
1165
+ t_ntoken_per_chunk, pure_audio_len - added_audio_len
1166
+ )
1167
+ ).expand(3, -1)
1168
+ + audio_start_idx
1169
+ ).split(1, dim=1)
1170
+ else:
1171
+ audio_llm_pos_ids_list = []
1172
+ added_audio_len += min(
1173
+ t_ntoken_per_chunk, pure_audio_len - added_audio_len
1174
+ )
1175
+ llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1176
+ if added_audio_len < pure_audio_len:
1177
+ new_src_item.extend(
1178
+ (pure_audio_len - added_audio_len) * [audio_token_id]
1179
+ )
1180
+ audio_llm_pos_ids_list = (
1181
+ torch.arange(pure_audio_len - added_audio_len).expand(3, -1)
1182
+ + llm_pos_ids_list[-1].max()
1183
+ + 1
1184
+ ).split(1, dim=1)
1185
+ llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1186
+ audio_idx += 1
1187
+ video_idx += 1
1188
+ # move to the next token
1189
+ idx += len(new_src_item) - new_src_item_len
1190
+
1191
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1)
1192
+ mrope_position_delta = (
1193
+ torch.cat(llm_pos_ids_list, dim=1).max() + 1 - len(src_item)
1194
+ )
1195
+ llm_positions = llm_positions[:, context_len:seq_len]
1196
+
1197
+ return llm_positions, mrope_position_delta
1198
+
1199
def get_multimodal_embeddings(self, **kwargs: object) -> MultiModalEmbeddings:
1200
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
1201
if not mm_input_by_modality:
vllm/model_executor/models/qwen2_5_vl.py
+129
-1
@@ -34,7 +34,7 @@ import torch
34
import torch.nn as nn
35
import torch.nn.functional as F
36
from einops import rearrange
37
-from transformers import BatchFeature
37
+from transformers import BatchFeature, PretrainedConfig
38
from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor
39
from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
40
Qwen2_5_VLConfig,
@@ -79,6 +79,7 @@ from .interfaces import (
79
MultiModalEmbeddings,
80
SupportsEagle3,
81
SupportsLoRA,
82
+ SupportsMRoPE,
83
SupportsMultiModal,
84
SupportsMultiModalPruning,
85
SupportsPP,
@@ -1053,6 +1054,7 @@ class Qwen2_5_VLForConditionalGeneration(
1054
SupportsQuant,
1055
SupportsEagle3,
1056
SupportsMultiModalPruning,
1057
+ SupportsMRoPE,
1058
):
1059
packed_modules_mapping = {
1060
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
@@ -1073,6 +1075,132 @@ class Qwen2_5_VLForConditionalGeneration(
1075
1076
supports_encoder_tp_data = True
1077
1078
+ @classmethod
1079
+ def get_mrope_input_positions(
1080
+ cls,
1081
+ input_tokens: list[int],
1082
+ hf_config: PretrainedConfig,
1083
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
1084
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
1085
+ second_per_grid_ts: list[float],
1086
+ context_len: int = 0,
1087
+ seq_len: Optional[int] = None,
1088
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1089
+ use_audio_in_video: bool = False,
1090
+ ) -> tuple[torch.Tensor, int]:
1091
+ """Get mrope input positions and delta value."""
1092
+
1093
+ image_token_id = hf_config.image_token_id
1094
+ video_token_id = hf_config.video_token_id
1095
+ vision_start_token_id = hf_config.vision_start_token_id
1096
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
1097
+ tokens_per_second = getattr(hf_config.vision_config, "tokens_per_second", 1.0)
1098
+
1099
+ input_tokens_tensor = torch.tensor(input_tokens)
1100
+ vision_start_indices = torch.argwhere(
1101
+ input_tokens_tensor == vision_start_token_id
1102
+ ).squeeze(1)
1103
+ vision_tokens = input_tokens_tensor[vision_start_indices + 1]
1104
+ image_nums = (vision_tokens == image_token_id).sum()
1105
+ video_nums = (vision_tokens == video_token_id).sum()
1106
+ llm_pos_ids_list: list = []
1107
+
1108
+ st = 0
1109
+ remain_images, remain_videos = image_nums, video_nums
1110
+
1111
+ image_index, video_index = 0, 0
1112
+ for _ in range(image_nums + video_nums):
1113
+ video_second_per_grid_t = 0.0
1114
+ if remain_images > 0:
1115
+ try:
1116
+ ed_image = input_tokens.index(image_token_id, st)
1117
+ except ValueError:
1118
+ ed_image = len(input_tokens) + 1
1119
+ else:
1120
+ ed_image = len(input_tokens) + 1
1121
+ if remain_videos > 0:
1122
+ try:
1123
+ ed_video = input_tokens.index(video_token_id, st)
1124
+ except ValueError:
1125
+ ed_video = len(input_tokens) + 1
1126
+ else:
1127
+ ed_video = len(input_tokens) + 1
1128
+ if ed_image < ed_video:
1129
+ t, h, w = (
1130
+ image_grid_thw[image_index][0],
1131
+ image_grid_thw[image_index][1],
1132
+ image_grid_thw[image_index][2],
1133
+ )
1134
+ image_index += 1
1135
+ remain_images -= 1
1136
+ ed = ed_image
1137
+ else:
1138
+ t, h, w = (
1139
+ video_grid_thw[video_index][0],
1140
+ video_grid_thw[video_index][1],
1141
+ video_grid_thw[video_index][2],
1142
+ )
1143
+ video_second_per_grid_t = 1.0
1144
+ if second_per_grid_ts:
1145
+ video_second_per_grid_t = second_per_grid_ts[video_index]
1146
+ video_index += 1
1147
+ remain_videos -= 1
1148
+ ed = ed_video
1149
+
1150
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1151
+ t,
1152
+ h // spatial_merge_size,
1153
+ w // spatial_merge_size,
1154
+ )
1155
+ text_len = ed - st
1156
+
1157
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1158
+ llm_pos_ids_list.append(
1159
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1160
+ )
1161
+
1162
+ t_index = (
1163
+ (
1164
+ torch.arange(llm_grid_t)
1165
+ .view(-1, 1)
1166
+ .expand(-1, llm_grid_h * llm_grid_w)
1167
+ * video_second_per_grid_t
1168
+ * tokens_per_second
1169
+ )
1170
+ .long()
1171
+ .flatten()
1172
+ )
1173
+
1174
+ h_index = (
1175
+ torch.arange(llm_grid_h)
1176
+ .view(1, -1, 1)
1177
+ .expand(llm_grid_t, -1, llm_grid_w)
1178
+ .flatten()
1179
+ )
1180
+ w_index = (
1181
+ torch.arange(llm_grid_w)
1182
+ .view(1, 1, -1)
1183
+ .expand(llm_grid_t, llm_grid_h, -1)
1184
+ .flatten()
1185
+ )
1186
+ llm_pos_ids_list.append(
1187
+ torch.stack([t_index, h_index, w_index]) + text_len + st_idx
1188
+ )
1189
+ st = ed + llm_grid_t * llm_grid_h * llm_grid_w
1190
+
1191
+ if st < len(input_tokens):
1192
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1193
+ text_len = len(input_tokens) - st
1194
+ llm_pos_ids_list.append(
1195
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1196
+ )
1197
+
1198
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1199
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1200
+ llm_positions = llm_positions[:, context_len:seq_len]
1201
+
1202
+ return llm_positions, mrope_position_delta
1203
+
1204
@classmethod
1205
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
1206
if modality.startswith("image"):
vllm/model_executor/models/qwen3_vl.py
+113
-2
@@ -33,7 +33,7 @@ import numpy as np
33
import torch
34
import torch.nn as nn
35
import torch.nn.functional as F
36
-from transformers import BatchFeature
36
+from transformers import BatchFeature, PretrainedConfig
37
from transformers.models.qwen2_vl import Qwen2VLImageProcessorFast
38
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
39
smart_resize as image_smart_resize,
@@ -84,6 +84,7 @@ from vllm.utils import is_list_of
84
from .interfaces import (
85
MultiModalEmbeddings,
86
SupportsLoRA,
87
+ SupportsMRoPE,
88
SupportsMultiModal,
89
SupportsPP,
90
)
@@ -1174,7 +1175,7 @@ class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
1175
dummy_inputs=Qwen3VLDummyInputsBuilder,
1176
)
1177
class Qwen3VLForConditionalGeneration(
1177
- nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP
1178
+ nn.Module, SupportsMultiModal, SupportsLoRA, SupportsPP, SupportsMRoPE
1179
):
1180
packed_modules_mapping = {
1181
"qkv_proj": [
@@ -1480,6 +1481,116 @@ class Qwen3VLForConditionalGeneration(
1481
)
1482
return mm_input_by_modality
1483
1484
+ @classmethod
1485
+ def get_mrope_input_positions(
1486
+ cls,
1487
+ input_tokens: list[int],
1488
+ hf_config: PretrainedConfig,
1489
+ image_grid_thw: Union[list[list[int]], torch.Tensor],
1490
+ video_grid_thw: Union[list[list[int]], torch.Tensor],
1491
+ context_len: int = 0,
1492
+ seq_len: Optional[int] = None,
1493
+ second_per_grid_ts: Optional[list[float]] = None,
1494
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1495
+ use_audio_in_video: bool = False,
1496
+ ) -> tuple[torch.Tensor, int]:
1497
+ """Get mrope input positions and delta value."""
1498
+
1499
+ video_grid_thw = [[1, h, w] for t, h, w in video_grid_thw for _ in range(t)]
1500
+
1501
+ image_token_id = hf_config.image_token_id
1502
+ video_token_id = hf_config.video_token_id
1503
+ vision_start_token_id = hf_config.vision_start_token_id
1504
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
1505
+
1506
+ input_tokens_tensor = torch.tensor(input_tokens)
1507
+ vision_start_indices = torch.argwhere(
1508
+ input_tokens_tensor == vision_start_token_id
1509
+ ).squeeze(1)
1510
+ vision_tokens = input_tokens_tensor[vision_start_indices + 1]
1511
+ image_nums = (vision_tokens == image_token_id).sum()
1512
+ video_nums = (vision_tokens == video_token_id).sum()
1513
+ llm_pos_ids_list: list = []
1514
+
1515
+ st = 0
1516
+ remain_images, remain_videos = image_nums, video_nums
1517
+
1518
+ image_index, video_index = 0, 0
1519
+ for _ in range(image_nums + video_nums):
1520
+ if image_token_id in input_tokens and remain_images > 0:
1521
+ ed_image = input_tokens.index(image_token_id, st)
1522
+ else:
1523
+ ed_image = len(input_tokens) + 1
1524
+ if video_token_id in input_tokens and remain_videos > 0:
1525
+ ed_video = input_tokens.index(video_token_id, st)
1526
+ else:
1527
+ ed_video = len(input_tokens) + 1
1528
+ if ed_image < ed_video:
1529
+ t, h, w = (
1530
+ image_grid_thw[image_index][0],
1531
+ image_grid_thw[image_index][1],
1532
+ image_grid_thw[image_index][2],
1533
+ )
1534
+ image_index += 1
1535
+ remain_images -= 1
1536
+ ed = ed_image
1537
+ else:
1538
+ t, h, w = (
1539
+ video_grid_thw[video_index][0],
1540
+ video_grid_thw[video_index][1],
1541
+ video_grid_thw[video_index][2],
1542
+ )
1543
+ video_index += 1
1544
+ remain_videos -= 1
1545
+ ed = ed_video
1546
+
1547
+ llm_grid_t, llm_grid_h, llm_grid_w = (
1548
+ t,
1549
+ h // spatial_merge_size,
1550
+ w // spatial_merge_size,
1551
+ )
1552
+ text_len = ed - st
1553
+
1554
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1555
+ llm_pos_ids_list.append(
1556
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1557
+ )
1558
+
1559
+ t_index = (
1560
+ torch.arange(llm_grid_t)
1561
+ .view(-1, 1)
1562
+ .expand(-1, llm_grid_h * llm_grid_w)
1563
+ .flatten()
1564
+ )
1565
+ h_index = (
1566
+ torch.arange(llm_grid_h)
1567
+ .view(1, -1, 1)
1568
+ .expand(llm_grid_t, -1, llm_grid_w)
1569
+ .flatten()
1570
+ )
1571
+ w_index = (
1572
+ torch.arange(llm_grid_w)
1573
+ .view(1, 1, -1)
1574
+ .expand(llm_grid_t, llm_grid_h, -1)
1575
+ .flatten()
1576
+ )
1577
+ llm_pos_ids_list.append(
1578
+ torch.stack([t_index, h_index, w_index]) + text_len + st_idx
1579
+ )
1580
+ st = ed + llm_grid_t * llm_grid_h * llm_grid_w
1581
+
1582
+ if st < len(input_tokens):
1583
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
1584
+ text_len = len(input_tokens) - st
1585
+ llm_pos_ids_list.append(
1586
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx
1587
+ )
1588
+
1589
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
1590
+ mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
1591
+ llm_positions = llm_positions[:, context_len:seq_len]
1592
+ return llm_positions, mrope_position_delta
1593
+
1594
def get_language_model(self) -> torch.nn.Module:
1595
return self.language_model
1596
vllm/model_executor/models/utils.py
+8
@@ -410,6 +410,14 @@ def _embedding_count_expression(embeddings: NestedTensors) -> str:
410
return " + ".join(_embedding_count_expression(inner) for inner in embeddings)
411
412
413
+def split_list_into_ranges(lst: torch.Tensor, interval: int) -> list[list[int]]:
414
+ ranges: list[list[int]] = [[] for _ in range((max(lst) // interval) + 1)]
415
+ for num in lst:
416
+ index = num // interval
417
+ ranges[index].append(num)
418
+ return ranges
419
+
420
+
421
def _merge_multimodal_embeddings(
422
inputs_embeds: torch.Tensor,
423
multimodal_embeddings: NestedTensors,
vllm/v1/worker/gpu_model_runner.py
+14
-25
@@ -875,30 +875,19 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
875
if mm_input.get("use_audio_in_video") is True:
876
use_audio_in_video = True
877
878
- if supports_mrope(self.get_model()):
879
- req_state.mrope_positions, req_state.mrope_position_delta = (
880
- self.model.get_mrope_input_positions(
881
- req_state.prompt_token_ids,
882
- hf_config=self.model_config.hf_config,
883
- image_grid_thw=image_grid_thw,
884
- video_grid_thw=video_grid_thw,
885
- second_per_grid_ts=second_per_grid_ts,
886
- audio_feature_lengths=audio_feature_lengths,
887
- use_audio_in_video=use_audio_in_video,
888
- )
889
- )
890
- else:
891
- req_state.mrope_positions, req_state.mrope_position_delta = (
892
- MRotaryEmbedding.get_input_positions_tensor(
893
- req_state.prompt_token_ids,
894
- hf_config=self.model_config.hf_config,
895
- image_grid_thw=image_grid_thw,
896
- video_grid_thw=video_grid_thw,
897
- second_per_grid_ts=second_per_grid_ts,
898
- audio_feature_lengths=audio_feature_lengths,
899
- use_audio_in_video=use_audio_in_video,
900
- )
878
+ assert supports_mrope(self.get_model()), "M-RoPE support is not implemented."
879
+
880
+ req_state.mrope_positions, req_state.mrope_position_delta = (
881
+ self.model.get_mrope_input_positions(
882
+ req_state.prompt_token_ids,
883
+ hf_config=self.model_config.hf_config,
884
+ image_grid_thw=image_grid_thw,
885
+ video_grid_thw=video_grid_thw,
886
+ second_per_grid_ts=second_per_grid_ts,
887
+ audio_feature_lengths=audio_feature_lengths,
888
+ use_audio_in_video=use_audio_in_video,
889
)
890
+ )
891
892
def _extract_mm_kwargs(
893
self,
@@ -2900,7 +2889,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2889
logger.info("Loading drafter model...")
2890
self.drafter.load_model(self.model)
2891
if self.use_aux_hidden_state_outputs:
2903
- if not supports_eagle3(self.model):
2892
+ if not supports_eagle3(self.get_model()):
2893
raise RuntimeError(
2894
"Model does not support EAGLE3 interface but "
2895
"aux_hidden_state_outputs was requested"
@@ -2928,7 +2917,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2917
prepare_communication_buffer_for_model(self.model)
2918
2919
self.is_multimodal_pruning_enabled = (
2931
- supports_multimodal_pruning(self.model)
2920
+ supports_multimodal_pruning(self.get_model())
2921
and self.model_config.multimodal_config.is_multimodal_pruning_enabled()
2922
)
2923