A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Feb 5, 2025 at 00:46 UTC
233df6f5c4520ae57e4a24acfbaedcc9ce166074
7 files changed
+66
-27
tests/entrypoints/openai/test_metrics.py
+1
@@ -205,6 +205,7 @@ EXPECTED_METRICS_V1 = [
205
"vllm:gpu_cache_usage_perc",
206
"vllm:prompt_tokens_total",
207
"vllm:generation_tokens_total",
208
+ "vllm:request_success_total",
209
"vllm:request_prompt_tokens_sum",
210
"vllm:request_prompt_tokens_bucket",
211
"vllm:request_prompt_tokens_count",
vllm/v1/engine/__init__.py
+19
-2
@@ -15,6 +15,23 @@ if TYPE_CHECKING:
15
from vllm.sampling_params import SamplingParams
16
17
18
+class RequestFinishedReason(enum.IntEnum):
19
+ """
20
+ Reason a request finished - stop, length, or abort.
21
+
22
+ stop - a stop string was emitted
23
+ length - max_tokens was consumed, or max_model_len was reached
24
+ abort - aborted for another reason
25
+
26
+ """
27
+ STOP = 0
28
+ LENGTH = 1
29
+ ABORT = 2
30
+
31
+ def __str__(self):
32
+ return self.name.lower()
33
+
34
+
35
@dataclass
36
class EngineCoreRequest:
37
@@ -45,7 +62,7 @@ class EngineCoreOutput(
62
request_id: str
63
new_token_ids: List[int]
64
finished: bool
48
- finish_reason: Optional[str] = None
65
+ finish_reason: Optional[RequestFinishedReason] = None
66
stop_reason: Union[int, str, None] = None
67
68
@@ -56,7 +73,7 @@ class EngineCoreOutputs(
73
gc=False): # type: ignore[call-arg]
74
75
#NOTE(Nick): We could consider ways to make this more compact,
59
- # e.g. columnwise layout and using an int enum for finish/stop reason
76
+ # e.g. columnwise layout
77
78
# [num_reqs]
79
outputs: List[EngineCoreOutput]
vllm/v1/engine/detokenizer.py
+5
-4
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8
from vllm.sampling_params import RequestOutputKind
9
from vllm.transformers_utils.detokenizer_utils import (
10
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
11
-from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
11
+from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
12
+ RequestFinishedReason)
13
14
logger = init_logger(__name__)
15
@@ -18,7 +19,7 @@ class DetokenizerOutput:
19
output_text: str
20
token_ids: List[int]
21
finished: bool
21
- finish_reason: Optional[str] = None
22
+ finish_reason: Optional[RequestFinishedReason] = None
23
stop_reason: Union[int, str, None] = None
24
25
@@ -147,13 +148,13 @@ class IncrementalDetokenizer:
148
stop_str, truncate_to = stop
149
if truncate_to != -1:
150
self.output_text = self.output_text[:truncate_to]
150
- finish_reason = "stop" # TODO: use constant
151
+ finish_reason = RequestFinishedReason.STOP
152
stop_reason = stop_str
153
154
# TODO: handle stop_token_ids here too?
155
156
# 3) Update the RequestOutput object with the new text.
156
- finished = bool(finish_reason)
157
+ finished = finish_reason is not None
158
if self.output_kind == RequestOutputKind.FINAL_ONLY \
159
and not finished:
160
return None
vllm/v1/engine/output_processor.py
+12
-10
@@ -161,8 +161,10 @@ class OutputProcessor:
161
engine_core_output)
162
163
# 3) Create and handle RequestOutput objects.
164
- if request_output := self._make_request_output(
165
- req_state, detokenizer_output):
164
+ if detokenizer_output is not None:
165
+ request_output = self._make_request_output(
166
+ req_state, detokenizer_output)
167
+
168
if req_state.queue is not None:
169
# AsyncLLM: put into queue for handling by generate().
170
req_state.queue.put_nowait(request_output)
@@ -172,6 +174,8 @@ class OutputProcessor:
174
175
# Free completed requests.
176
if request_output.finished:
177
+ assert detokenizer_output.finish_reason is not None
178
+
179
self.request_states.pop(req_id)
180
if not engine_core_output.finished:
181
# If req not finished in EngineCore, but Detokenizer
@@ -180,7 +184,8 @@ class OutputProcessor:
184
185
# Track per-request stats
186
iteration_stats.update_from_finished_request(
183
- request_output, req_state.stats)
187
+ detokenizer_output.finish_reason, request_output,
188
+ req_state.stats)
189
190
return OutputProcessorOutput(
191
request_outputs=request_outputs,
@@ -191,12 +196,8 @@ class OutputProcessor:
196
@staticmethod
197
def _make_request_output(
198
request_state: RequestState,
194
- detokenizer_output: Optional[DetokenizerOutput],
195
- ) -> Optional[RequestOutput]:
196
-
197
- if detokenizer_output is None:
198
- return None
199
-
199
+ detokenizer_output: DetokenizerOutput,
200
+ ) -> RequestOutput:
201
request_output = RequestOutput.new(
202
request_state.request_id,
203
request_state.prompt,
@@ -207,7 +208,8 @@ class OutputProcessor:
208
)
209
if detokenizer_output.finished:
210
completion_output = request_output.outputs[0]
210
- completion_output.finish_reason = detokenizer_output.finish_reason
211
+ completion_output.finish_reason = str(
212
+ detokenizer_output.finish_reason)
213
completion_output.stop_reason = detokenizer_output.stop_reason
214
215
return request_output
vllm/v1/metrics/loggers.py
+14
-1
@@ -2,13 +2,14 @@
2
3
import time
4
from abc import ABC, abstractmethod
5
-from typing import List
5
+from typing import Dict, List
6
7
import numpy as np
8
import prometheus_client
9
10
from vllm.config import ModelConfig
11
from vllm.logger import init_logger
12
+from vllm.v1.engine import RequestFinishedReason
13
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
14
15
logger = init_logger(__name__)
@@ -116,6 +117,17 @@ class PrometheusStatLogger(StatLoggerBase):
117
documentation="Number of generation tokens processed.",
118
labelnames=labelnames).labels(*labelvalues)
119
120
+ self.counter_request_success: Dict[RequestFinishedReason,
121
+ prometheus_client.Counter] = {}
122
+ counter_request_success_base = prometheus_client.Counter(
123
+ name="vllm:request_success_total",
124
+ documentation="Count of successfully processed requests.",
125
+ labelnames=labelnames + ["finished_reason"])
126
+ for reason in RequestFinishedReason:
127
+ self.counter_request_success[
128
+ reason] = counter_request_success_base.labels(*(labelvalues +
129
+ [str(reason)]))
130
+
131
self.histogram_num_prompt_tokens_request = \
132
prometheus_client.Histogram(
133
name="vllm:request_prompt_tokens",
@@ -163,6 +175,7 @@ class PrometheusStatLogger(StatLoggerBase):
175
iteration_stats.num_generation_tokens)
176
177
for finished_request in iteration_stats.finished_requests:
178
+ self.counter_request_success[finished_request.finish_reason].inc()
179
self.histogram_num_prompt_tokens_request.observe(
180
finished_request.num_prompt_tokens)
181
self.histogram_num_generation_tokens_request.observe(
vllm/v1/metrics/stats.py
+7
-3
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7
if TYPE_CHECKING:
8
from vllm.outputs import RequestOutput
9
- from vllm.v1.engine import EngineCoreOutput
9
+ from vllm.v1.engine import EngineCoreOutput, RequestFinishedReason
10
11
12
@dataclass
@@ -32,6 +32,7 @@ class RequestStateStats:
32
class FinishedRequestStats:
33
"""Stats associated with a finished request."""
34
35
+ finish_reason: "RequestFinishedReason"
36
num_prompt_tokens: int = 0
37
num_generation_tokens: int = 0
38
@@ -73,8 +74,11 @@ class IterationStats:
74
request_state_stats.num_generation_tokens += num_new_generation_tokens
75
request_state_stats.last_token_time = now
76
76
- def update_from_finished_request(self, request_output: "RequestOutput",
77
+ def update_from_finished_request(self,
78
+ finish_reason: "RequestFinishedReason",
79
+ request_output: "RequestOutput",
80
request_state_stats: RequestStateStats):
81
self.finished_requests.append(
79
- FinishedRequestStats(len(request_output.prompt_token_ids),
82
+ FinishedRequestStats(finish_reason,
83
+ len(request_output.prompt_token_ids),
84
request_state_stats.num_generation_tokens))
vllm/v1/request.py
+8
-7
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
6
from vllm.lora.request import LoRARequest
7
from vllm.sampling_params import SamplingParams
8
from vllm.sequence import RequestMetrics
9
-from vllm.v1.engine import EngineCoreRequest
9
+from vllm.v1.engine import EngineCoreRequest, RequestFinishedReason
10
from vllm.v1.utils import ConstantList
11
12
if TYPE_CHECKING:
@@ -109,7 +109,7 @@ class Request:
109
def is_finished(self) -> bool:
110
return RequestStatus.is_finished(self.status)
111
112
- def get_finished_reason(self) -> Union[str, None]:
112
+ def get_finished_reason(self) -> Union[RequestFinishedReason, None]:
113
return RequestStatus.get_finished_reason(self.status)
114
115
def has_encoder_inputs(self) -> bool:
@@ -149,7 +149,8 @@ class RequestStatus(enum.IntEnum):
149
return status > RequestStatus.PREEMPTED
150
151
@staticmethod
152
- def get_finished_reason(status: "RequestStatus") -> Union[str, None]:
152
+ def get_finished_reason(
153
+ status: "RequestStatus") -> Union[RequestFinishedReason, None]:
154
return _FINISHED_REASON_MAP.get(status)
155
156
@@ -158,8 +159,8 @@ class RequestStatus(enum.IntEnum):
159
# are longer than the model's length cap. Therefore, the stop
160
# reason should also be "length" as in OpenAI API.
161
_FINISHED_REASON_MAP = {
161
- RequestStatus.FINISHED_STOPPED: "stop",
162
- RequestStatus.FINISHED_LENGTH_CAPPED: "length",
163
- RequestStatus.FINISHED_ABORTED: "abort",
164
- RequestStatus.FINISHED_IGNORED: "length",
162
+ RequestStatus.FINISHED_STOPPED: RequestFinishedReason.STOP,
163
+ RequestStatus.FINISHED_LENGTH_CAPPED: RequestFinishedReason.LENGTH,
164
+ RequestStatus.FINISHED_ABORTED: RequestFinishedReason.ABORT,
165
+ RequestStatus.FINISHED_IGNORED: RequestFinishedReason.LENGTH,
166
}