@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3d09e592a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Misc] Shorten `FinishReason` enum and use constant strings (#12760)

Nick Hill committed Feb 4, 2025 at 22:43 UTC 3d09e592a860982acef0edef858078d28d393e84
5 files changed +25 -21
vllm/v1/engine/__init__.py
+9 -3
@@ -14,11 +14,17 @@ if TYPE_CHECKING:
14 from vllm.multimodal.inputs import PlaceholderRange
15 from vllm.sampling_params import SamplingParams
16
17 +# These are possible values of RequestOutput.finish_reason,
18 +# so form part of the external API.
19 +FINISH_REASON_STRINGS = ("stop", "length", "abort")
20
18 -class RequestFinishedReason(enum.IntEnum):
21 +
22 +class FinishReason(enum.IntEnum):
23 """
24 Reason a request finished - stop, length, or abort.
25
26 + Int rather than Str for more compact serialization.
27 +
28 stop - a stop string was emitted
29 length - max_tokens was consumed, or max_model_len was reached
30 abort - aborted for another reason
@@ -29,7 +35,7 @@ class RequestFinishedReason(enum.IntEnum):
35 ABORT = 2
36
37 def __str__(self):
32 - return self.name.lower()
38 + return FINISH_REASON_STRINGS[self.value]
39
40
41 @dataclass
@@ -62,7 +68,7 @@ class EngineCoreOutput(
68 request_id: str
69 new_token_ids: List[int]
70 finished: bool
65 - finish_reason: Optional[RequestFinishedReason] = None
71 + finish_reason: Optional[FinishReason] = None
72 stop_reason: Union[int, str, None] = None
73
74
vllm/v1/engine/detokenizer.py
+3 -4
@@ -8,8 +8,7 @@ from vllm.logger import init_logger
8 from vllm.sampling_params import RequestOutputKind
9 from vllm.transformers_utils.detokenizer_utils import (
10 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
11 -from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
12 - RequestFinishedReason)
11 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
12
13 logger = init_logger(__name__)
14
@@ -19,7 +18,7 @@ class DetokenizerOutput:
18 output_text: str
19 token_ids: List[int]
20 finished: bool
22 - finish_reason: Optional[RequestFinishedReason] = None
21 + finish_reason: Optional[FinishReason] = None
22 stop_reason: Union[int, str, None] = None
23
24
@@ -148,7 +147,7 @@ class IncrementalDetokenizer:
147 stop_str, truncate_to = stop
148 if truncate_to != -1:
149 self.output_text = self.output_text[:truncate_to]
151 - finish_reason = RequestFinishedReason.STOP
150 + finish_reason = FinishReason.STOP
151 stop_reason = stop_str
152
153 # TODO: handle stop_token_ids here too?
vllm/v1/metrics/loggers.py
+3 -3
@@ -9,7 +9,7 @@ import prometheus_client
9
10 from vllm.config import ModelConfig
11 from vllm.logger import init_logger
12 -from vllm.v1.engine import RequestFinishedReason
12 +from vllm.v1.engine import FinishReason
13 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
14
15 logger = init_logger(__name__)
@@ -117,13 +117,13 @@ class PrometheusStatLogger(StatLoggerBase):
117 documentation="Number of generation tokens processed.",
118 labelnames=labelnames).labels(*labelvalues)
119
120 - self.counter_request_success: Dict[RequestFinishedReason,
120 + self.counter_request_success: Dict[FinishReason,
121 prometheus_client.Counter] = {}
122 counter_request_success_base = prometheus_client.Counter(
123 name="vllm:request_success_total",
124 documentation="Count of successfully processed requests.",
125 labelnames=labelnames + ["finished_reason"])
126 - for reason in RequestFinishedReason:
126 + for reason in FinishReason:
127 self.counter_request_success[
128 reason] = counter_request_success_base.labels(*(labelvalues +
129 [str(reason)]))
vllm/v1/metrics/stats.py
+3 -4
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7 if TYPE_CHECKING:
8 from vllm.outputs import RequestOutput
9 - from vllm.v1.engine import EngineCoreOutput, RequestFinishedReason
9 + from vllm.v1.engine import EngineCoreOutput, FinishReason
10
11
12 @dataclass
@@ -32,7 +32,7 @@ class RequestStateStats:
32 class FinishedRequestStats:
33 """Stats associated with a finished request."""
34
35 - finish_reason: "RequestFinishedReason"
35 + finish_reason: "FinishReason"
36 num_prompt_tokens: int = 0
37 num_generation_tokens: int = 0
38
@@ -74,8 +74,7 @@ class IterationStats:
74 request_state_stats.num_generation_tokens += num_new_generation_tokens
75 request_state_stats.last_token_time = now
76
77 - def update_from_finished_request(self,
78 - finish_reason: "RequestFinishedReason",
77 + def update_from_finished_request(self, finish_reason: "FinishReason",
78 request_output: "RequestOutput",
79 request_state_stats: RequestStateStats):
80 self.finished_requests.append(
vllm/v1/request.py
+7 -7
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
6 from vllm.lora.request import LoRARequest
7 from vllm.sampling_params import SamplingParams
8 from vllm.sequence import RequestMetrics
9 -from vllm.v1.engine import EngineCoreRequest, RequestFinishedReason
9 +from vllm.v1.engine import EngineCoreRequest, FinishReason
10 from vllm.v1.utils import ConstantList
11
12 if TYPE_CHECKING:
@@ -109,7 +109,7 @@ class Request:
109 def is_finished(self) -> bool:
110 return RequestStatus.is_finished(self.status)
111
112 - def get_finished_reason(self) -> Union[RequestFinishedReason, None]:
112 + def get_finished_reason(self) -> Union[FinishReason, None]:
113 return RequestStatus.get_finished_reason(self.status)
114
115 def has_encoder_inputs(self) -> bool:
@@ -150,7 +150,7 @@ class RequestStatus(enum.IntEnum):
150
151 @staticmethod
152 def get_finished_reason(
153 - status: "RequestStatus") -> Union[RequestFinishedReason, None]:
153 + status: "RequestStatus") -> Union[FinishReason, None]:
154 return _FINISHED_REASON_MAP.get(status)
155
156
@@ -159,8 +159,8 @@ class RequestStatus(enum.IntEnum):
159 # are longer than the model's length cap. Therefore, the stop
160 # reason should also be "length" as in OpenAI API.
161 _FINISHED_REASON_MAP = {
162 - RequestStatus.FINISHED_STOPPED: RequestFinishedReason.STOP,
163 - RequestStatus.FINISHED_LENGTH_CAPPED: RequestFinishedReason.LENGTH,
164 - RequestStatus.FINISHED_ABORTED: RequestFinishedReason.ABORT,
165 - RequestStatus.FINISHED_IGNORED: RequestFinishedReason.LENGTH,
162 + RequestStatus.FINISHED_STOPPED: FinishReason.STOP,
163 + RequestStatus.FINISHED_LENGTH_CAPPED: FinishReason.LENGTH,
164 + RequestStatus.FINISHED_ABORTED: FinishReason.ABORT,
165 + RequestStatus.FINISHED_IGNORED: FinishReason.LENGTH,
166 }