A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Fix mrope_position_delta in non-last prefill chunk (#10403)
Signed-off-by: imkero <kerorek@outlook.com>
电脑星人 committed
Nov 17, 2024 at 16:50 UTC
80d85c5d7bc33ce0ae210ebad3c45e4361b57640
1 file changed
+1
-1
vllm/model_executor/layers/rotary_embedding.py
+1
-1
@@ -922,9 +922,9 @@ class MRotaryEmbedding(RotaryEmbedding):
922
torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
923
924
llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
925
- llm_positions = llm_positions[:, context_len:seq_len]
925
mrope_position_delta = (llm_positions.max() + 1 -
926
len(input_tokens)).item()
927
+ llm_positions = llm_positions[:, context_len:seq_len]
928
929
return llm_positions.tolist(), mrope_position_delta
930