A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] fix tool_parser error handling when serve a model not support it (#8709)
Yanyi Liu committed
Oct 6, 2024 at 20:51 UTC
fdf59d30eaf1a62979b2a13016b4f47f28f12f88
3 files changed
+31
-10
vllm/entrypoints/openai/serving_chat.py
+21
-5
@@ -302,10 +302,6 @@ class OpenAIServingChat(OpenAIServing):
302
finish_reason_sent = [False] * num_choices
303
num_prompt_tokens = 0
304
305
- tool_parsers: List[Optional[ToolParser]] = [
306
- self.tool_parser(tokenizer) if self.tool_parser else None
307
- ] * num_choices
308
-
305
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
306
tool_choice_function_name = request.tool_choice.function.name
307
else:
@@ -324,6 +320,21 @@ class OpenAIServingChat(OpenAIServing):
320
else:
321
previous_texts, all_previous_token_ids = None, None
322
323
+ # Prepare the tool parser if it's needed
324
+ try:
325
+ if tool_choice_auto and self.tool_parser:
326
+ tool_parsers: List[Optional[ToolParser]] = [
327
+ self.tool_parser(tokenizer)
328
+ ] * num_choices
329
+ else:
330
+ tool_parsers = [None] * num_choices
331
+ except RuntimeError as e:
332
+ logger.error("Error in tool parser creation: %s", e)
333
+ data = self.create_streaming_error_response(str(e))
334
+ yield f"data: {data}\n\n"
335
+ yield "data: [DONE]\n\n"
336
+ return
337
+
338
try:
339
async for res in result_generator:
340
if res.prompt_token_ids is not None:
@@ -704,7 +715,12 @@ class OpenAIServingChat(OpenAIServing):
715
or request.tool_choice is None) and self.enable_auto_tools \
716
and self.tool_parser:
717
707
- tool_parser = self.tool_parser(tokenizer)
718
+ try:
719
+ tool_parser = self.tool_parser(tokenizer)
720
+ except RuntimeError as e:
721
+ logger.error("Error in tool parser creation: %s", e)
722
+ return self.create_error_response(str(e))
723
+
724
tool_call_info = tool_parser.extract_tool_calls(
725
output.text, request=request)
726
tools_called = tool_call_info.tools_called
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+4
-4
@@ -50,10 +50,10 @@ class Hermes2ProToolParser(ToolParser):
50
raise ValueError(
51
"The model tokenizer must be passed to the ToolParser "
52
"constructor during construction.")
53
- self.tool_call_start_token_id: int = self.model_tokenizer.vocab[
54
- self.tool_call_start_token]
55
- self.tool_call_end_token_id: int = self.model_tokenizer.vocab[
56
- self.tool_call_end_token]
53
+ self.tool_call_start_token_id: int = self.model_tokenizer.vocab.get(
54
+ self.tool_call_start_token, None)
55
+ self.tool_call_end_token_id: int = self.model_tokenizer.vocab.get(
56
+ self.tool_call_end_token, None)
57
if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
58
raise RuntimeError(
59
"Hermes 2 Pro Tool parser could not locate tool call start/end "
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+6
-1
@@ -61,8 +61,13 @@ class MistralToolParser(ToolParser):
61
self.streamed_args_for_tool: List[str] = [
62
] # map what has been streamed for each tool so far to a list
63
self.bot_token = "[TOOL_CALLS]"
64
- self.bot_token_id = self.model_tokenizer.get_vocab()[self.bot_token]
64
+ self.bot_token_id = self.model_tokenizer.get_vocab().get(
65
+ self.bot_token, None)
66
self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
67
+ if not self.bot_token_id:
68
+ raise RuntimeError(
69
+ "Mistral Tool Parser could not locate the tool call token in "
70
+ "the tokenizer!")
71
72
def extract_tool_calls(
73
self,