| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Prometheus metrics for the parsers.""" |
| 4 | |
| 5 | from __future__ import annotations |
| 6 | |
| 7 | from enum import Enum |
| 8 | from itertools import product |
| 9 | from typing import cast |
| 10 | |
| 11 | from prometheus_client import REGISTRY, Counter |
| 12 | |
| 13 | _model_name: str | None = None |
| 14 | |
| 15 | _TOOL_CALL_PARSER_INVOCATIONS_TOTAL = "vllm:tool_call_parser_invocations_total" |
| 16 | _tool_call_parser_invocations: Counter | None = None |
| 17 | |
| 18 | |
| 19 | class ToolCallOutcome(Enum): |
| 20 | TOOL_CALL = "tool_call" |
| 21 | NO_TOOL_CALL = "no_tool_call" |
| 22 | |
| 23 | |
| 24 | class RequestType(Enum): |
| 25 | CHAT_COMPLETIONS = "chat_completions" |
| 26 | RESPONSES = "responses" |
| 27 | OTHER = "other" |
| 28 | |
| 29 | |
| 30 | def init_parser_metrics(*, model_name: str) -> None: |
| 31 | """Lazily register parser metrics and cache the shared model label.""" |
| 32 | global _model_name |
| 33 | _model_name = model_name |
| 34 | |
| 35 | global _tool_call_parser_invocations |
| 36 | try: |
| 37 | _tool_call_parser_invocations = Counter( |
| 38 | name=_TOOL_CALL_PARSER_INVOCATIONS_TOTAL, |
| 39 | documentation=( |
| 40 | "Total number of ToolParser invocations. " |
| 41 | "Non-streaming increments once per choice; " |
| 42 | "streaming increments once per delta." |
| 43 | ), |
| 44 | labelnames=["model_name", "mode", "outcome", "request_type"], |
| 45 | ) |
| 46 | except ValueError: |
| 47 | _tool_call_parser_invocations = cast( |
| 48 | Counter, |
| 49 | REGISTRY._names_to_collectors[_TOOL_CALL_PARSER_INVOCATIONS_TOTAL], |
| 50 | ) |
| 51 | |
| 52 | for mode, outcome, request_type in product( |
| 53 | ("streaming", "non_streaming"), |
| 54 | ToolCallOutcome, |
| 55 | RequestType, |
| 56 | ): |
| 57 | _tool_call_parser_invocations.labels( |
| 58 | model_name=_model_name, |
| 59 | mode=mode, |
| 60 | outcome=outcome.value, |
| 61 | request_type=request_type.value, |
| 62 | ) |
| 63 | |
| 64 | |
| 65 | def record_tool_parser_invocation( |
| 66 | *, |
| 67 | is_tool_called: bool | Exception, |
| 68 | is_streaming: bool, |
| 69 | request: object, |
| 70 | ) -> None: |
| 71 | """Increment the tool-call parser invocation counter when registered. |
| 72 | Currently parser failures are treated as no tool calls. |
| 73 | |
| 74 | TODO: To accurately track parser failures, add a new ToolCallOutcome and |
| 75 | more importantly, ensure exceptions are propagated out of the ToolParsers |
| 76 | instead of being caught internally. This would require going through |
| 77 | ToolParser implementation on a case-by-case basis. |
| 78 | """ |
| 79 | if _tool_call_parser_invocations is None: |
| 80 | return |
| 81 | |
| 82 | from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest |
| 83 | from vllm.entrypoints.openai.responses.protocol import ResponsesRequest |
| 84 | |
| 85 | match request: |
| 86 | case ChatCompletionRequest(): |
| 87 | request_type = RequestType.CHAT_COMPLETIONS |
| 88 | case ResponsesRequest(): |
| 89 | request_type = RequestType.RESPONSES |
| 90 | case _: |
| 91 | request_type = RequestType.OTHER |
| 92 | |
| 93 | match is_tool_called: |
| 94 | case bool(): |
| 95 | outcome = ( |
| 96 | ToolCallOutcome.TOOL_CALL |
| 97 | if is_tool_called |
| 98 | else ToolCallOutcome.NO_TOOL_CALL |
| 99 | ) |
| 100 | case _: |
| 101 | outcome = ToolCallOutcome.NO_TOOL_CALL |
| 102 | |
| 103 | _tool_call_parser_invocations.labels( |
| 104 | model_name=_model_name, |
| 105 | mode="streaming" if is_streaming else "non_streaming", |
| 106 | outcome=outcome.value, |
| 107 | request_type=request_type.value, |
| 108 | ).inc() |