A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Don't skip special tokens with hermes-style tool calling (#25281)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Sep 19, 2025 at 21:33 UTC
a36c675817867235d368faf7e8d81e0ed3333d9c
1 file changed
+9
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+9
@@ -98,6 +98,15 @@ class Hermes2ProToolParser(ToolParser):
98
else:
99
return delta_text
100
101
+ def adjust_request(
102
+ self, request: ChatCompletionRequest) -> ChatCompletionRequest:
103
+ if request.tools and request.tool_choice != 'none':
104
+ # do not skip special tokens because the tool_call tokens are
105
+ # marked "special" in some models. Since they are skipped
106
+ # prior to the call to the tool parser, it breaks tool calling.
107
+ request.skip_special_tokens = False
108
+ return request
109
+
110
def extract_tool_calls(
111
self,
112
model_output: str,