A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Bugfix] Fix Qwen2.5-Omni M-RoPE position ids generation (#16878)
Signed-off-by: imkero <kerorek@outlook.com>
Kero Liang committed
Apr 27, 2025 at 01:41 UTC
de7eb10ce4fba23d204e06ecec4261669fab884a
1 file changed
+13
-15
vllm/model_executor/layers/rotary_embedding.py
+13
-15
@@ -1209,6 +1209,7 @@ class MRotaryEmbedding(RotaryEmbedding):
1209
video_token_id = thinker_config.video_token_index
1210
audio_start_token_id = thinker_config.audio_start_token_id
1211
audio_end_token_id = thinker_config.audio_end_token_id
1212
+ vision_start_token_id = thinker_config.vision_start_token_id
1213
vision_end_token_id = thinker_config.vision_end_token_id
1214
seconds_per_chunk = thinker_config.seconds_per_chunk
1215
spatial_merge_size = thinker_config.vision_config.spatial_merge_size
@@ -1238,8 +1239,15 @@ class MRotaryEmbedding(RotaryEmbedding):
1239
if src_item[idx] not in [
1240
audio_token_id, video_token_id, image_token_id
1241
]:
1241
- if src_item[idx] == vision_end_token_id and use_audio_in_video:
1242
- start_idx -= 1
1242
+ if use_audio_in_video and idx > 0:
1243
+ if src_item[idx] == vision_end_token_id and \
1244
+ src_item[idx - 1] == audio_end_token_id:
1245
+ # processing the <|audio_eos|> before <|vision_eos|>
1246
+ start_idx -= 1
1247
+ elif src_item[idx] == audio_start_token_id and \
1248
+ src_item[idx - 1] == vision_start_token_id:
1249
+ # processing the <|audio_bos|> after <|vision_eos|>
1250
+ start_idx -= 1
1251
new_src_item.append(src_item[idx])
1252
llm_pos_ids = torch.tensor([start_idx],
1253
dtype=torch.long).expand(3, -1)
@@ -1297,11 +1305,6 @@ class MRotaryEmbedding(RotaryEmbedding):
1305
tokens_per_second).long()
1306
t_index_split_chunk = cls._split_list_into_ranges(
1307
t_index, t_ntoken_per_chunk)
1300
- new_src_item.extend([audio_start_token_id])
1301
- start_idx -= 1
1302
- llm_pos_ids_list.extend([
1303
- torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
1304
- ] * 1)
1308
place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
1309
pure_audio_len = place_num - 2
1310
added_audio_len = 0
@@ -1312,7 +1315,7 @@ class MRotaryEmbedding(RotaryEmbedding):
1315
new_src_item.extend([video_token_id] *
1316
vision_ntoken_per_chunk)
1317
vision_llm_pos_ids_list = cls._get_llm_pos_ids_for_vision(
1315
- start_idx + 1, video_idx, spatial_merge_size, t_chunk,
1318
+ start_idx, video_idx, spatial_merge_size, t_chunk,
1319
grid_hs, grid_ws).split(1, dim=1)
1320
llm_pos_ids_list.extend(vision_llm_pos_ids_list)
1321
new_src_item.extend(
@@ -1320,13 +1323,13 @@ class MRotaryEmbedding(RotaryEmbedding):
1323
added_audio_len) * [audio_token_id])
1324
audio_start_idx = start_idx if len(
1325
audio_llm_pos_ids_list
1323
- ) == 0 else audio_llm_pos_ids_list[-1][0].item()
1326
+ ) == 0 else audio_llm_pos_ids_list[-1][0].item() + 1
1327
if min(t_ntoken_per_chunk,
1328
pure_audio_len - added_audio_len) > 0:
1329
audio_llm_pos_ids_list = (torch.arange(
1330
min(t_ntoken_per_chunk, pure_audio_len -
1331
added_audio_len)).expand(3, -1) +
1329
- audio_start_idx + 1).split(
1332
+ audio_start_idx).split(
1333
1, dim=1)
1334
else:
1335
audio_llm_pos_ids_list = []
@@ -1341,11 +1344,6 @@ class MRotaryEmbedding(RotaryEmbedding):
1344
3, -1) + llm_pos_ids_list[-1].max() + 1).split(
1345
1, dim=1)
1346
llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1344
- llm_pos_ids_list.extend([
1345
- torch.tensor(
1346
- [llm_pos_ids_list[-1].max() + 1] * 3).unsqueeze(1)
1347
- ] * 1)
1348
- new_src_item.extend([audio_end_token_id])
1347
audio_idx += 1
1348
video_idx += 1
1349
# move to the next token