@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c01f70641
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Core] Optimize `SequenceStatus.is_finished` by switching to IntEnum (#5974)

Antoni Baum committed Jun 29, 2024 at 05:47 UTC 7c01f706418d593b3cf23d2ec9110dca7151c539
1 file changed +11 -14
vllm/sequence.py
+11 -14
@@ -39,24 +39,21 @@ PromptLogprobs = List[Optional[Dict[int, Logprob]]]
39 SampleLogprobs = List[Dict[int, Logprob]]
40
41
42 -class SequenceStatus(enum.Enum):
42 +class SequenceStatus(enum.IntEnum):
43 """Status of a sequence."""
44 - WAITING = enum.auto()
45 - RUNNING = enum.auto()
46 - SWAPPED = enum.auto()
47 - FINISHED_STOPPED = enum.auto()
48 - FINISHED_LENGTH_CAPPED = enum.auto()
49 - FINISHED_ABORTED = enum.auto()
50 - FINISHED_IGNORED = enum.auto()
44 + WAITING = 0
45 + RUNNING = 1
46 + SWAPPED = 2
47 + # Note: anything after SWAPPED (2) will be considered
48 + # as a finished status.
49 + FINISHED_STOPPED = 3
50 + FINISHED_LENGTH_CAPPED = 4
51 + FINISHED_ABORTED = 5
52 + FINISHED_IGNORED = 6
53
54 @staticmethod
55 def is_finished(status: "SequenceStatus") -> bool:
54 - return status in [
55 - SequenceStatus.FINISHED_STOPPED,
56 - SequenceStatus.FINISHED_LENGTH_CAPPED,
57 - SequenceStatus.FINISHED_ABORTED,
58 - SequenceStatus.FINISHED_IGNORED,
59 - ]
56 + return status > SequenceStatus.SWAPPED
57
58 @staticmethod
59 def get_finished_reason(status: "SequenceStatus") -> Union[str, None]: