@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 233df6f5c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Add request_success_total counter, labelled with finish reason (#12579)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Feb 5, 2025 at 00:46 UTC 233df6f5c4520ae57e4a24acfbaedcc9ce166074
7 files changed +66 -27
tests/entrypoints/openai/test_metrics.py
+1
@@ -205,6 +205,7 @@ EXPECTED_METRICS_V1 = [
205 "vllm:gpu_cache_usage_perc",
206 "vllm:prompt_tokens_total",
207 "vllm:generation_tokens_total",
208 + "vllm:request_success_total",
209 "vllm:request_prompt_tokens_sum",
210 "vllm:request_prompt_tokens_bucket",
211 "vllm:request_prompt_tokens_count",
vllm/v1/engine/__init__.py
+19 -2
@@ -15,6 +15,23 @@ if TYPE_CHECKING:
15 from vllm.sampling_params import SamplingParams
16
17
18 +class RequestFinishedReason(enum.IntEnum):
19 + """
20 + Reason a request finished - stop, length, or abort.
21 +
22 + stop - a stop string was emitted
23 + length - max_tokens was consumed, or max_model_len was reached
24 + abort - aborted for another reason
25 +
26 + """
27 + STOP = 0
28 + LENGTH = 1
29 + ABORT = 2
30 +
31 + def __str__(self):
32 + return self.name.lower()
33 +
34 +
35 @dataclass
36 class EngineCoreRequest:
37
@@ -45,7 +62,7 @@ class EngineCoreOutput(
62 request_id: str
63 new_token_ids: List[int]
64 finished: bool
48 - finish_reason: Optional[str] = None
65 + finish_reason: Optional[RequestFinishedReason] = None
66 stop_reason: Union[int, str, None] = None
67
68
@@ -56,7 +73,7 @@ class EngineCoreOutputs(
73 gc=False): # type: ignore[call-arg]
74
75 #NOTE(Nick): We could consider ways to make this more compact,
59 - # e.g. columnwise layout and using an int enum for finish/stop reason
76 + # e.g. columnwise layout
77
78 # [num_reqs]
79 outputs: List[EngineCoreOutput]
vllm/v1/engine/detokenizer.py
+5 -4
@@ -8,7 +8,8 @@ from vllm.logger import init_logger
8 from vllm.sampling_params import RequestOutputKind
9 from vllm.transformers_utils.detokenizer_utils import (
10 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
11 -from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
11 +from vllm.v1.engine import (EngineCoreOutput, EngineCoreRequest,
12 + RequestFinishedReason)
13
14 logger = init_logger(__name__)
15
@@ -18,7 +19,7 @@ class DetokenizerOutput:
19 output_text: str
20 token_ids: List[int]
21 finished: bool
21 - finish_reason: Optional[str] = None
22 + finish_reason: Optional[RequestFinishedReason] = None
23 stop_reason: Union[int, str, None] = None
24
25
@@ -147,13 +148,13 @@ class IncrementalDetokenizer:
148 stop_str, truncate_to = stop
149 if truncate_to != -1:
150 self.output_text = self.output_text[:truncate_to]
150 - finish_reason = "stop" # TODO: use constant
151 + finish_reason = RequestFinishedReason.STOP
152 stop_reason = stop_str
153
154 # TODO: handle stop_token_ids here too?
155
156 # 3) Update the RequestOutput object with the new text.
156 - finished = bool(finish_reason)
157 + finished = finish_reason is not None
158 if self.output_kind == RequestOutputKind.FINAL_ONLY \
159 and not finished:
160 return None
vllm/v1/engine/output_processor.py
+12 -10
@@ -161,8 +161,10 @@ class OutputProcessor:
161 engine_core_output)
162
163 # 3) Create and handle RequestOutput objects.
164 - if request_output := self._make_request_output(
165 - req_state, detokenizer_output):
164 + if detokenizer_output is not None:
165 + request_output = self._make_request_output(
166 + req_state, detokenizer_output)
167 +
168 if req_state.queue is not None:
169 # AsyncLLM: put into queue for handling by generate().
170 req_state.queue.put_nowait(request_output)
@@ -172,6 +174,8 @@ class OutputProcessor:
174
175 # Free completed requests.
176 if request_output.finished:
177 + assert detokenizer_output.finish_reason is not None
178 +
179 self.request_states.pop(req_id)
180 if not engine_core_output.finished:
181 # If req not finished in EngineCore, but Detokenizer
@@ -180,7 +184,8 @@ class OutputProcessor:
184
185 # Track per-request stats
186 iteration_stats.update_from_finished_request(
183 - request_output, req_state.stats)
187 + detokenizer_output.finish_reason, request_output,
188 + req_state.stats)
189
190 return OutputProcessorOutput(
191 request_outputs=request_outputs,
@@ -191,12 +196,8 @@ class OutputProcessor:
196 @staticmethod
197 def _make_request_output(
198 request_state: RequestState,
194 - detokenizer_output: Optional[DetokenizerOutput],
195 - ) -> Optional[RequestOutput]:
196 -
197 - if detokenizer_output is None:
198 - return None
199 -
199 + detokenizer_output: DetokenizerOutput,
200 + ) -> RequestOutput:
201 request_output = RequestOutput.new(
202 request_state.request_id,
203 request_state.prompt,
@@ -207,7 +208,8 @@ class OutputProcessor:
208 )
209 if detokenizer_output.finished:
210 completion_output = request_output.outputs[0]
210 - completion_output.finish_reason = detokenizer_output.finish_reason
211 + completion_output.finish_reason = str(
212 + detokenizer_output.finish_reason)
213 completion_output.stop_reason = detokenizer_output.stop_reason
214
215 return request_output
vllm/v1/metrics/loggers.py
+14 -1
@@ -2,13 +2,14 @@
2
3 import time
4 from abc import ABC, abstractmethod
5 -from typing import List
5 +from typing import Dict, List
6
7 import numpy as np
8 import prometheus_client
9
10 from vllm.config import ModelConfig
11 from vllm.logger import init_logger
12 +from vllm.v1.engine import RequestFinishedReason
13 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
14
15 logger = init_logger(__name__)
@@ -116,6 +117,17 @@ class PrometheusStatLogger(StatLoggerBase):
117 documentation="Number of generation tokens processed.",
118 labelnames=labelnames).labels(*labelvalues)
119
120 + self.counter_request_success: Dict[RequestFinishedReason,
121 + prometheus_client.Counter] = {}
122 + counter_request_success_base = prometheus_client.Counter(
123 + name="vllm:request_success_total",
124 + documentation="Count of successfully processed requests.",
125 + labelnames=labelnames + ["finished_reason"])
126 + for reason in RequestFinishedReason:
127 + self.counter_request_success[
128 + reason] = counter_request_success_base.labels(*(labelvalues +
129 + [str(reason)]))
130 +
131 self.histogram_num_prompt_tokens_request = \
132 prometheus_client.Histogram(
133 name="vllm:request_prompt_tokens",
@@ -163,6 +175,7 @@ class PrometheusStatLogger(StatLoggerBase):
175 iteration_stats.num_generation_tokens)
176
177 for finished_request in iteration_stats.finished_requests:
178 + self.counter_request_success[finished_request.finish_reason].inc()
179 self.histogram_num_prompt_tokens_request.observe(
180 finished_request.num_prompt_tokens)
181 self.histogram_num_generation_tokens_request.observe(
vllm/v1/metrics/stats.py
+7 -3
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List
6
7 if TYPE_CHECKING:
8 from vllm.outputs import RequestOutput
9 - from vllm.v1.engine import EngineCoreOutput
9 + from vllm.v1.engine import EngineCoreOutput, RequestFinishedReason
10
11
12 @dataclass
@@ -32,6 +32,7 @@ class RequestStateStats:
32 class FinishedRequestStats:
33 """Stats associated with a finished request."""
34
35 + finish_reason: "RequestFinishedReason"
36 num_prompt_tokens: int = 0
37 num_generation_tokens: int = 0
38
@@ -73,8 +74,11 @@ class IterationStats:
74 request_state_stats.num_generation_tokens += num_new_generation_tokens
75 request_state_stats.last_token_time = now
76
76 - def update_from_finished_request(self, request_output: "RequestOutput",
77 + def update_from_finished_request(self,
78 + finish_reason: "RequestFinishedReason",
79 + request_output: "RequestOutput",
80 request_state_stats: RequestStateStats):
81 self.finished_requests.append(
79 - FinishedRequestStats(len(request_output.prompt_token_ids),
82 + FinishedRequestStats(finish_reason,
83 + len(request_output.prompt_token_ids),
84 request_state_stats.num_generation_tokens))
vllm/v1/request.py
+8 -7
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, List, Optional, Union
6 from vllm.lora.request import LoRARequest
7 from vllm.sampling_params import SamplingParams
8 from vllm.sequence import RequestMetrics
9 -from vllm.v1.engine import EngineCoreRequest
9 +from vllm.v1.engine import EngineCoreRequest, RequestFinishedReason
10 from vllm.v1.utils import ConstantList
11
12 if TYPE_CHECKING:
@@ -109,7 +109,7 @@ class Request:
109 def is_finished(self) -> bool:
110 return RequestStatus.is_finished(self.status)
111
112 - def get_finished_reason(self) -> Union[str, None]:
112 + def get_finished_reason(self) -> Union[RequestFinishedReason, None]:
113 return RequestStatus.get_finished_reason(self.status)
114
115 def has_encoder_inputs(self) -> bool:
@@ -149,7 +149,8 @@ class RequestStatus(enum.IntEnum):
149 return status > RequestStatus.PREEMPTED
150
151 @staticmethod
152 - def get_finished_reason(status: "RequestStatus") -> Union[str, None]:
152 + def get_finished_reason(
153 + status: "RequestStatus") -> Union[RequestFinishedReason, None]:
154 return _FINISHED_REASON_MAP.get(status)
155
156
@@ -158,8 +159,8 @@ class RequestStatus(enum.IntEnum):
159 # are longer than the model's length cap. Therefore, the stop
160 # reason should also be "length" as in OpenAI API.
161 _FINISHED_REASON_MAP = {
161 - RequestStatus.FINISHED_STOPPED: "stop",
162 - RequestStatus.FINISHED_LENGTH_CAPPED: "length",
163 - RequestStatus.FINISHED_ABORTED: "abort",
164 - RequestStatus.FINISHED_IGNORED: "length",
162 + RequestStatus.FINISHED_STOPPED: RequestFinishedReason.STOP,
163 + RequestStatus.FINISHED_LENGTH_CAPPED: RequestFinishedReason.LENGTH,
164 + RequestStatus.FINISHED_ABORTED: RequestFinishedReason.ABORT,
165 + RequestStatus.FINISHED_IGNORED: RequestFinishedReason.LENGTH,
166 }