A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[V1][Bugfix] Fix handing of `second_per_grid_ts` for Qwen2-VL & Qwen2.5-VL (#14548)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Mar 10, 2025 at 09:03 UTC
dea985aef01bde5f58fb0a9a6afb38ead4433e55
1 file changed
+10
-5
vllm/model_executor/layers/rotary_embedding.py
+10
-5
@@ -923,14 +923,19 @@ class MRotaryEmbedding(RotaryEmbedding):
923
def get_input_positions(
924
input_tokens: List[int],
925
hf_config: PretrainedConfig,
926
- image_grid_thw: Union[List[List[int]], torch.Tensor],
927
- video_grid_thw: Union[List[List[int]], torch.Tensor],
928
- second_per_grid_ts: Optional[List[float]] = None,
926
+ image_grid_thw: Optional[Union[List[List[int]], torch.Tensor]],
927
+ video_grid_thw: Optional[Union[List[List[int]], torch.Tensor]],
928
+ second_per_grid_ts: Optional[List[float]],
929
context_len: int = 0,
930
seq_len: Optional[int] = None,
931
) -> Tuple[List[List[int]], int]:
932
"""Get mrope input positions and delta value."""
933
934
+ image_grid_thw = [] if image_grid_thw is None else image_grid_thw
935
+ video_grid_thw = [] if video_grid_thw is None else video_grid_thw
936
+ second_per_grid_ts = [] if second_per_grid_ts is None else \
937
+ second_per_grid_ts
938
+
939
llm_positions, mrope_position_delta = \
940
MRotaryEmbedding.get_input_positions_tensor(
941
input_tokens=input_tokens,
@@ -950,7 +955,7 @@ class MRotaryEmbedding(RotaryEmbedding):
955
hf_config: PretrainedConfig,
956
image_grid_thw: Union[List[List[int]], torch.Tensor],
957
video_grid_thw: Union[List[List[int]], torch.Tensor],
953
- second_per_grid_ts: Optional[List[float]] = None,
958
+ second_per_grid_ts: List[float],
959
context_len: int = 0,
960
seq_len: Optional[int] = None,
961
) -> Tuple[torch.Tensor, int]:
@@ -1006,7 +1011,7 @@ class MRotaryEmbedding(RotaryEmbedding):
1011
video_grid_thw[video_index][2],
1012
)
1013
video_second_per_grid_t = 1.0
1009
- if second_per_grid_ts is not None:
1014
+ if second_per_grid_ts:
1015
video_second_per_grid_t = second_per_grid_ts[video_index]
1016
video_index += 1
1017
remain_videos -= 1