A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Misc] Shorten `FinishReason` enum and use constant strings (#12760)
Nick Hill committed
Feb 4, 2025 at 22:43 UTC
3d09e592a860982acef0edef858078d28d393e84
5 files changed
+25
-21
vllm/v1/engine/__init__.py
+9
-3
@@ -14,11 +14,17 @@ if TYPE_CHECKING:
14
from vllm.multimodal.inputs import PlaceholderRange
15
from vllm.sampling_params import SamplingParams
16
17
+# These are possible values of RequestOutput.finish_reason,
18
+# so form part of the external API.
19
+FINISH_REASON_STRINGS = ("stop", "length", "abort")
20
18
-class RequestFinishedReason(enum.IntEnum):
21
+
22
+class FinishReason(enum.IntEnum):
23
"""
24
Reason a request finished - stop, length, or abort.
25
26
+ Int rather than Str for more compact serialization.
27
+
28
stop - a stop string was emitted
29
length - max_tokens was consumed, or max_model_len was reached
30
abort - aborted for another reason
@@ -29,7 +35,7 @@ class RequestFinishedReason(enum.IntEnum):
35
ABORT = 2
36
37
def __str__(self):
32
- return self.name.lower()
38
+ return FINISH_REASON_STRINGS[self.value]
39
40
41
@dataclass
@@ -62,7 +68,7 @@ class EngineCoreOutput(
68
request_id: str
69
new_token_ids: List[int]
70
finished: bool
65
- finish_reason: Optional[RequestFinishedReason] = None
71
+ finish_reason: Optional[FinishReason] = None
72
stop_reason: Union[int, str, None] = None
73
74
vllm/v1/engine/detokenizer.py
+3
-4
@@ -8,8 +8,7 @@ from vllm.logger import init_logger
8
from vllm.sampling_params import RequestOutputKind
9
from vllm.transformers_utils.detokenizer_utils import (
10
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
11
-from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
12
- RequestFinishedReason)
11
+from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
12
13
logger = init_logger(__name__)
14
@@ -19,7 +18,7 @@ class DetokenizerOutput:
18
output_text: str
19
token_ids: List[int]
20
finished: bool
22
- finish_reason: Optional[RequestFinishedReason] = None
21
+ finish_reason: Optional[FinishReason] = None
22
stop_reason: Union[int, str, None] = None
23
24
@@ -148,7 +147,7 @@ class IncrementalDetokenizer:
147
stop_str, truncate_to = stop
148
if truncate_to != -1:
149
self.output_text = self.output_text[:truncate_to]
151
- finish_reason = RequestFinishedReason.STOP
150
+ finish_reason = FinishReason.STOP
151
stop_reason = stop_str
152
153
# TODO: handle stop_token_ids here too?
vllm/v1/metrics/loggers.py
+3
-3
@@ -9,7 +9,7 @@ import prometheus_client
9
10
from vllm.config import ModelConfig
11
from vllm.logger import init_logger
12
-from vllm.v1.engine import RequestFinishedReason
12
+from vllm.v1.engine import FinishReason
13
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
14
15
logger = init_logger(__name__)
@@ -117,13 +117,13 @@ class PrometheusStatLogger(StatLoggerBase):
117
documentation="Number of generation tokens processed.",
118
labelnames=labelnames).labels(*labelvalues)
119
120
- self.counter_request_success: Dict[RequestFinishedReason,
120
+ self.counter_request_success: Dict[FinishReason,
121
prometheus_client.Counter] = {}
122
counter_request_success_base = prometheus_client.Counter(
123
name="vllm:request_success_total",
124
documentation="Count of successfully processed requests.",
125
labelnames=labelnames + ["finished_reason"])
126
- for reason in RequestFinishedReason:
126
+ for reason in FinishReason:
127
self.counter_request_success[
128
reason] = counter_request_success_base.labels(*(labelvalues +
129
[str(reason)]))
vllm/v1/metrics/stats.py
+3
-4
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7
if TYPE_CHECKING:
8
from vllm.outputs import RequestOutput
9
- from vllm.v1.engine import EngineCoreOutput, RequestFinishedReason
9
+ from vllm.v1.engine import EngineCoreOutput, FinishReason
10
11
12
@dataclass
@@ -32,7 +32,7 @@ class RequestStateStats:
32
class FinishedRequestStats:
33
"""Stats associated with a finished request."""
34
35
- finish_reason: "RequestFinishedReason"
35
+ finish_reason: "FinishReason"
36
num_prompt_tokens: int = 0
37
num_generation_tokens: int = 0
38
@@ -74,8 +74,7 @@ class IterationStats:
74
request_state_stats.num_generation_tokens += num_new_generation_tokens
75
request_state_stats.last_token_time = now
76
77
- def update_from_finished_request(self,
78
- finish_reason: "RequestFinishedReason",
77
+ def update_from_finished_request(self, finish_reason: "FinishReason",
78
request_output: "RequestOutput",
79
request_state_stats: RequestStateStats):
80
self.finished_requests.append(
vllm/v1/request.py
+7
-7
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
6
from vllm.lora.request import LoRARequest
7
from vllm.sampling_params import SamplingParams
8
from vllm.sequence import RequestMetrics
9
-from vllm.v1.engine import EngineCoreRequest, RequestFinishedReason
9
+from vllm.v1.engine import EngineCoreRequest, FinishReason
10
from vllm.v1.utils import ConstantList
11
12
if TYPE_CHECKING:
@@ -109,7 +109,7 @@ class Request:
109
def is_finished(self) -> bool:
110
return RequestStatus.is_finished(self.status)
111
112
- def get_finished_reason(self) -> Union[RequestFinishedReason, None]:
112
+ def get_finished_reason(self) -> Union[FinishReason, None]:
113
return RequestStatus.get_finished_reason(self.status)
114
115
def has_encoder_inputs(self) -> bool:
@@ -150,7 +150,7 @@ class RequestStatus(enum.IntEnum):
150
151
@staticmethod
152
def get_finished_reason(
153
- status: "RequestStatus") -> Union[RequestFinishedReason, None]:
153
+ status: "RequestStatus") -> Union[FinishReason, None]:
154
return _FINISHED_REASON_MAP.get(status)
155
156
@@ -159,8 +159,8 @@ class RequestStatus(enum.IntEnum):
159
# are longer than the model's length cap. Therefore, the stop
160
# reason should also be "length" as in OpenAI API.
161
_FINISHED_REASON_MAP = {
162
- RequestStatus.FINISHED_STOPPED: RequestFinishedReason.STOP,
163
- RequestStatus.FINISHED_LENGTH_CAPPED: RequestFinishedReason.LENGTH,
164
- RequestStatus.FINISHED_ABORTED: RequestFinishedReason.ABORT,
165
- RequestStatus.FINISHED_IGNORED: RequestFinishedReason.LENGTH,
162
+ RequestStatus.FINISHED_STOPPED: FinishReason.STOP,
163
+ RequestStatus.FINISHED_LENGTH_CAPPED: FinishReason.LENGTH,
164
+ RequestStatus.FINISHED_ABORTED: FinishReason.ABORT,
165
+ RequestStatus.FINISHED_IGNORED: FinishReason.LENGTH,
166
}