A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
Tool parser regex timeout handling (#18960)
Signed-off-by: Will Eaton <weaton@redhat.com>
Will Eaton committed
May 30, 2025 at 17:02 UTC
1dab4d5718243687acd9edc0ccaa0c94ddc43934
5 files changed
+82
-5
tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py
+25
-1
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from unittest.mock import MagicMock
3
+from unittest.mock import MagicMock, patch
4
5
import pytest
6
@@ -191,3 +191,27 @@ def test_streaming_tool_call_with_large_steps():
191
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
192
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
193
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
194
+
195
+
196
+@pytest.mark.parametrize("streaming", [False])
197
+def test_regex_timeout_handling(streaming: bool):
198
+ """test regex timeout is handled gracefully"""
199
+ mock_tokenizer = MagicMock()
200
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser(
201
+ "llama4_pythonic")(mock_tokenizer)
202
+
203
+ fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
204
+
205
+ # create a mock regex that raises TimeoutError
206
+ mock_regex = MagicMock()
207
+ mock_regex.match.side_effect = TimeoutError("Regex timeout")
208
+
209
+ with patch.object(tool_parser, 'TOOL_CALL_REGEX', mock_regex):
210
+ content, tool_calls = run_tool_extraction(tool_parser,
211
+ fake_problematic_input,
212
+ streaming=streaming)
213
+
214
+ # should treat as regular text when regex times out
215
+ assert content == fake_problematic_input
216
+ assert len(tool_calls) == 0
217
+ mock_regex.match.assert_called_once()
tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py
+25
-1
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from unittest.mock import MagicMock
3
+from unittest.mock import MagicMock, patch
4
5
import pytest
6
@@ -159,3 +159,27 @@ def test_streaming_tool_call_with_large_steps():
159
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
160
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
161
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
162
+
163
+
164
+@pytest.mark.parametrize("streaming", [False])
165
+def test_regex_timeout_handling(streaming: bool):
166
+ """test regex timeout is handled gracefully"""
167
+ mock_tokenizer = MagicMock()
168
+ tool_parser: ToolParser = ToolParserManager.get_tool_parser(
169
+ "llama4_pythonic")(mock_tokenizer)
170
+
171
+ fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
172
+
173
+ # create a mock regex that raises TimeoutError
174
+ mock_regex = MagicMock()
175
+ mock_regex.match.side_effect = TimeoutError("Regex timeout")
176
+
177
+ with patch.object(tool_parser, 'TOOL_CALL_REGEX', mock_regex):
178
+ content, tool_calls = run_tool_extraction(tool_parser,
179
+ fake_problematic_input,
180
+ streaming=streaming)
181
+
182
+ # should treat as regular text when regex times out
183
+ assert content == fake_problematic_input
184
+ assert len(tool_calls) == 0
185
+ mock_regex.match.assert_called_once()
vllm/entrypoints/openai/tool_parsers/llama4_pythonic_tool_parser.py
+14
-1
@@ -7,6 +7,7 @@ from typing import Any, Union
7
import regex as re
8
from transformers import PreTrainedTokenizerBase
9
10
+import vllm.envs as envs
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
DeltaFunctionCall, DeltaMessage,
13
DeltaToolCall,
@@ -64,7 +65,19 @@ class Llama4PythonicToolParser(ToolParser):
65
if model_output.startswith("<|python_start|>"):
66
model_output = model_output[len("<|python_start|>"):]
67
model_output = model_output.replace("<|python_end|>", "")
67
- if not (self.TOOL_CALL_REGEX.match(model_output)):
68
+
69
+ is_tool_call_pattern = False
70
+ try:
71
+ is_tool_call_pattern = self.TOOL_CALL_REGEX.match(
72
+ model_output,
73
+ timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS) is not None
74
+ except TimeoutError:
75
+ logger.warning(
76
+ "Regex timeout occurred when matching tool call pattern.")
77
+ logger.debug("Regex timeout occurred when matching user input: %s",
78
+ model_output)
79
+
80
+ if not is_tool_call_pattern:
81
return ExtractedToolCallInformation(tools_called=False,
82
tool_calls=[],
83
content=model_output)
vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py
+13
-2
@@ -8,6 +8,7 @@ from typing import Any, Union
8
import regex as re
9
from transformers import PreTrainedTokenizerBase
10
11
+import vllm.envs as envs
12
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13
DeltaFunctionCall, DeltaMessage,
14
DeltaToolCall,
@@ -61,8 +62,18 @@ class PythonicToolParser(ToolParser):
62
"""
63
Extract the tool calls from a complete model response.
64
"""
64
-
65
- if not (self.TOOL_CALL_REGEX.match(model_output)):
65
+ is_tool_call_pattern = False
66
+ try:
67
+ is_tool_call_pattern = self.TOOL_CALL_REGEX.match(
68
+ model_output,
69
+ timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS) is not None
70
+ except TimeoutError:
71
+ logger.warning(
72
+ "Regex timeout occurred when matching tool call pattern.")
73
+ logger.debug("Regex timeout occurred when matching user input: %s",
74
+ model_output)
75
+
76
+ if not is_tool_call_pattern:
77
return ExtractedToolCallInformation(tools_called=False,
78
tool_calls=[],
79
content=model_output)
vllm/envs.py
+5
@@ -119,6 +119,7 @@ if TYPE_CHECKING:
119
VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5557
120
VLLM_ALL2ALL_BACKEND: str = "naive"
121
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
122
+ VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
123
124
125
def get_default_cache_root():
@@ -828,6 +829,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
829
# This is used to prevent the kernel from running out of memory.
830
"VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE":
831
lambda: int(os.getenv("VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE", "163840")),
832
+
833
+ # Regex timeout for use by the vLLM tool parsing plugins.
834
+ "VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS":
835
+ lambda: int(os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")),
836
}
837
838
# --8<-- [end:env-vars-definition]