@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 80d85c5d7b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix mrope_position_delta in non-last prefill chunk (#10403)

Signed-off-by: imkero <kerorek@outlook.com>

电脑星人 committed Nov 17, 2024 at 16:50 UTC 80d85c5d7bc33ce0ae210ebad3c45e4361b57640
1 file changed +1 -1
vllm/model_executor/layers/rotary_embedding.py
+1 -1
@@ -922,9 +922,9 @@ class MRotaryEmbedding(RotaryEmbedding):
922 torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
923
924 llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
925 - llm_positions = llm_positions[:, context_len:seq_len]
925 mrope_position_delta = (llm_positions.max() + 1 -
926 len(input_tokens)).item()
927 + llm_positions = llm_positions[:, context_len:seq_len]
928
929 return llm_positions.tolist(), mrope_position_delta
930