A high-throughput and memory-efficient inference and serving engine for LLMs
[Frontend][Metrics] Add `vllm:tool_call_parser_invocations_total` Prometheus metric (#44448)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
yzong-rh committed
Jun 10, 2026 at 10:29 UTC
6ec7dcd64125ac13f6863718e854e127d2132ced
3 files changed
+167
-20
vllm/entrypoints/openai/api_server.py
+8
-1
@@ -13,7 +13,7 @@ import warnings
13
from argparse import Namespace
14
from collections.abc import AsyncIterator
15
from contextlib import asynccontextmanager
16
-from typing import Any
16
+from typing import Any, cast
17
18
import uvloop
19
from fastapi import FastAPI, HTTPException
@@ -322,6 +322,13 @@ async def init_app_state(
322
vllm_config.structured_outputs_config.enable_in_reasoning
323
)
324
325
+ if args.tool_call_parser is not None:
326
+ from vllm.parser.metrics import init_parser_metrics
327
+
328
+ init_parser_metrics(
329
+ model_name=cast(str, vllm_config.model_config.served_model_name)
330
+ )
331
+
332
if supported_tasks is None:
333
warnings.warn(
334
"The 'supported_tasks' parameter was not provided to "
vllm/parser/abstract_parser.py
+51
-19
@@ -35,6 +35,7 @@ from vllm.entrypoints.openai.engine.protocol import (
35
)
36
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
37
from vllm.logger import init_logger
38
+from vllm.parser.metrics import record_tool_parser_invocation
39
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
40
from vllm.tokenizers import TokenizerLike
41
from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser
@@ -276,7 +277,7 @@ class Parser:
277
def extract_tool_calls(
278
self,
279
model_output: str,
279
- request: ChatCompletionRequest,
280
+ request: ChatCompletionRequest | ResponsesRequest,
281
) -> ExtractedToolCallInformation:
282
"""
283
Extract tool calls from a complete model-generated string.
@@ -300,7 +301,7 @@ class Parser:
301
previous_token_ids: Sequence[int],
302
current_token_ids: Sequence[int],
303
delta_token_ids: Sequence[int],
303
- request: ChatCompletionRequest,
304
+ request: ChatCompletionRequest | ResponsesRequest,
305
) -> DeltaMessage | None:
306
"""
307
Extract tool calls from a streaming delta message.
@@ -514,9 +515,9 @@ class DelegatingParser(Parser):
515
and (request.tool_choice == "auto" or request.tool_choice is None)
516
):
517
# Automatic Tool Call Parsing
517
- tool_call_info = self._tool_parser.extract_tool_calls(
518
+ tool_call_info = self.extract_tool_calls(
519
content if content is not None else "",
519
- request=request, # type: ignore
520
+ request=request,
521
)
522
if tool_call_info is not None and tool_call_info.tools_called:
523
function_calls.extend(
@@ -590,9 +591,9 @@ class DelegatingParser(Parser):
591
elif is_auto_tool_choice:
592
# Automatic Tool Call Parsing (also used as fallback for
593
# required/named when supports_required_and_named=False)
593
- tool_call_info = tool_parser.extract_tool_calls(
594
+ tool_call_info = self.extract_tool_calls(
595
content if content is not None else "",
595
- request=request, # type: ignore
596
+ request=request,
597
)
598
if tool_call_info is not None and tool_call_info.tools_called:
599
tool_calls.extend(
@@ -644,13 +645,30 @@ class DelegatingParser(Parser):
645
def extract_tool_calls(
646
self,
647
model_output: str,
647
- request: ChatCompletionRequest,
648
+ request: ChatCompletionRequest | ResponsesRequest,
649
) -> ExtractedToolCallInformation:
650
if self._tool_parser is None:
651
return ExtractedToolCallInformation(
652
tools_called=False, tool_calls=[], content=model_output
653
)
653
- return self._tool_parser.extract_tool_calls(model_output, request)
654
+ result = None
655
+ is_tool_called: bool | Exception = False
656
+ try:
657
+ result = self._tool_parser.extract_tool_calls(
658
+ model_output,
659
+ request=request, # type: ignore[arg-type]
660
+ )
661
+ is_tool_called = bool(result.tools_called)
662
+ except Exception as e:
663
+ is_tool_called = e
664
+ raise
665
+ finally:
666
+ record_tool_parser_invocation(
667
+ is_tool_called=is_tool_called,
668
+ is_streaming=False,
669
+ request=request,
670
+ )
671
+ return result
672
673
def extract_tool_calls_streaming(
674
self,
@@ -660,19 +678,33 @@ class DelegatingParser(Parser):
678
previous_token_ids: Sequence[int],
679
current_token_ids: Sequence[int],
680
delta_token_ids: Sequence[int],
663
- request: ChatCompletionRequest,
681
+ request: ChatCompletionRequest | ResponsesRequest,
682
) -> DeltaMessage | None:
683
if self._tool_parser is None:
684
return None
667
- return self._tool_parser.extract_tool_calls_streaming(
668
- previous_text,
669
- current_text,
670
- delta_text,
671
- previous_token_ids,
672
- current_token_ids,
673
- delta_token_ids,
674
- request,
675
- )
685
+ result = None
686
+ is_tool_called: bool | Exception = False
687
+ try:
688
+ result = self._tool_parser.extract_tool_calls_streaming(
689
+ previous_text,
690
+ current_text,
691
+ delta_text,
692
+ previous_token_ids,
693
+ current_token_ids,
694
+ delta_token_ids,
695
+ request, # type: ignore[arg-type]
696
+ )
697
+ is_tool_called = bool(result and result.tool_calls)
698
+ except Exception as e:
699
+ is_tool_called = e
700
+ raise
701
+ finally:
702
+ record_tool_parser_invocation(
703
+ is_tool_called=is_tool_called,
704
+ is_streaming=True,
705
+ request=request,
706
+ )
707
+ return result
708
709
def _extract_tool_calls_streaming(
710
self,
@@ -731,7 +763,7 @@ class DelegatingParser(Parser):
763
previous_token_ids,
764
current_token_ids,
765
delta_token_ids,
734
- request, # type: ignore[arg-type]
766
+ request,
767
), False
768
769
def is_reasoning_end(self, input_ids: list[int]) -> bool:
vllm/parser/metrics.py
new
+108
@@ -0,0 +1,108 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Prometheus metrics for the parsers."""
4
+
5
+from __future__ import annotations
6
+
7
+from enum import Enum
8
+from itertools import product
9
+from typing import cast
10
+
11
+from prometheus_client import REGISTRY, Counter
12
+
13
+_model_name: str | None = None
14
+
15
+_TOOL_CALL_PARSER_INVOCATIONS_TOTAL = "vllm:tool_call_parser_invocations_total"
16
+_tool_call_parser_invocations: Counter | None = None
17
+
18
+
19
+class ToolCallOutcome(Enum):
20
+ TOOL_CALL = "tool_call"
21
+ NO_TOOL_CALL = "no_tool_call"
22
+
23
+
24
+class RequestType(Enum):
25
+ CHAT_COMPLETIONS = "chat_completions"
26
+ RESPONSES = "responses"
27
+ OTHER = "other"
28
+
29
+
30
+def init_parser_metrics(*, model_name: str) -> None:
31
+ """Lazily register parser metrics and cache the shared model label."""
32
+ global _model_name
33
+ _model_name = model_name
34
+
35
+ global _tool_call_parser_invocations
36
+ try:
37
+ _tool_call_parser_invocations = Counter(
38
+ name=_TOOL_CALL_PARSER_INVOCATIONS_TOTAL,
39
+ documentation=(
40
+ "Total number of ToolParser invocations. "
41
+ "Non-streaming increments once per choice; "
42
+ "streaming increments once per delta."
43
+ ),
44
+ labelnames=["model_name", "mode", "outcome", "request_type"],
45
+ )
46
+ except ValueError:
47
+ _tool_call_parser_invocations = cast(
48
+ Counter,
49
+ REGISTRY._names_to_collectors[_TOOL_CALL_PARSER_INVOCATIONS_TOTAL],
50
+ )
51
+
52
+ for mode, outcome, request_type in product(
53
+ ("streaming", "non_streaming"),
54
+ ToolCallOutcome,
55
+ RequestType,
56
+ ):
57
+ _tool_call_parser_invocations.labels(
58
+ model_name=_model_name,
59
+ mode=mode,
60
+ outcome=outcome.value,
61
+ request_type=request_type.value,
62
+ )
63
+
64
+
65
+def record_tool_parser_invocation(
66
+ *,
67
+ is_tool_called: bool | Exception,
68
+ is_streaming: bool,
69
+ request: object,
70
+) -> None:
71
+ """Increment the tool-call parser invocation counter when registered.
72
+ Currently parser failures are treated as no tool calls.
73
+
74
+ TODO: To accurately track parser failures, add a new ToolCallOutcome and
75
+ more importantly, ensure exceptions are propagated out of the ToolParsers
76
+ instead of being caught internally. This would require going through
77
+ ToolParser implementation on a case-by-case basis.
78
+ """
79
+ if _tool_call_parser_invocations is None:
80
+ return
81
+
82
+ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
83
+ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
84
+
85
+ match request:
86
+ case ChatCompletionRequest():
87
+ request_type = RequestType.CHAT_COMPLETIONS
88
+ case ResponsesRequest():
89
+ request_type = RequestType.RESPONSES
90
+ case _:
91
+ request_type = RequestType.OTHER
92
+
93
+ match is_tool_called:
94
+ case bool():
95
+ outcome = (
96
+ ToolCallOutcome.TOOL_CALL
97
+ if is_tool_called
98
+ else ToolCallOutcome.NO_TOOL_CALL
99
+ )
100
+ case _:
101
+ outcome = ToolCallOutcome.NO_TOOL_CALL
102
+
103
+ _tool_call_parser_invocations.labels(
104
+ model_name=_model_name,
105
+ mode="streaming" if is_streaming else "non_streaming",
106
+ outcome=outcome.value,
107
+ request_type=request_type.value,
108
+ ).inc()