A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[FIX] Explain why the finished_reason of ignored sequences are length (#1289)
Zhuohan Li committed
Oct 8, 2023 at 15:22 UTC
6b5296aa3ae632b8f2dcbc78579eb41b28e41068
1 file changed
+3
vllm/sequence.py
+3
@@ -35,6 +35,9 @@ class SequenceStatus(enum.Enum):
35
elif status == SequenceStatus.FINISHED_ABORTED:
36
finish_reason = "abort"
37
elif status == SequenceStatus.FINISHED_IGNORED:
38
+ # The ignored sequences are the sequences whose prompt lengths
39
+ # are longer than the model's length cap. Therefore, the stop
40
+ # reason should also be "length" as in OpenAI API.
41
finish_reason = "length"
42
else:
43
finish_reason = None