@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fdf59d30ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] fix tool_parser error handling when serve a model not support it (#8709)

Yanyi Liu committed Oct 6, 2024 at 20:51 UTC fdf59d30eaf1a62979b2a13016b4f47f28f12f88
3 files changed +31 -10
vllm/entrypoints/openai/serving_chat.py
+21 -5
@@ -302,10 +302,6 @@ class OpenAIServingChat(OpenAIServing):
302 finish_reason_sent = [False] * num_choices
303 num_prompt_tokens = 0
304
305 - tool_parsers: List[Optional[ToolParser]] = [
306 - self.tool_parser(tokenizer) if self.tool_parser else None
307 - ] * num_choices
308 -
305 if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
306 tool_choice_function_name = request.tool_choice.function.name
307 else:
@@ -324,6 +320,21 @@ class OpenAIServingChat(OpenAIServing):
320 else:
321 previous_texts, all_previous_token_ids = None, None
322
323 + # Prepare the tool parser if it's needed
324 + try:
325 + if tool_choice_auto and self.tool_parser:
326 + tool_parsers: List[Optional[ToolParser]] = [
327 + self.tool_parser(tokenizer)
328 + ] * num_choices
329 + else:
330 + tool_parsers = [None] * num_choices
331 + except RuntimeError as e:
332 + logger.error("Error in tool parser creation: %s", e)
333 + data = self.create_streaming_error_response(str(e))
334 + yield f"data: {data}\n\n"
335 + yield "data: [DONE]\n\n"
336 + return
337 +
338 try:
339 async for res in result_generator:
340 if res.prompt_token_ids is not None:
@@ -704,7 +715,12 @@ class OpenAIServingChat(OpenAIServing):
715 or request.tool_choice is None) and self.enable_auto_tools \
716 and self.tool_parser:
717
707 - tool_parser = self.tool_parser(tokenizer)
718 + try:
719 + tool_parser = self.tool_parser(tokenizer)
720 + except RuntimeError as e:
721 + logger.error("Error in tool parser creation: %s", e)
722 + return self.create_error_response(str(e))
723 +
724 tool_call_info = tool_parser.extract_tool_calls(
725 output.text, request=request)
726 tools_called = tool_call_info.tools_called
vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py
+4 -4
@@ -50,10 +50,10 @@ class Hermes2ProToolParser(ToolParser):
50 raise ValueError(
51 "The model tokenizer must be passed to the ToolParser "
52 "constructor during construction.")
53 - self.tool_call_start_token_id: int = self.model_tokenizer.vocab[
54 - self.tool_call_start_token]
55 - self.tool_call_end_token_id: int = self.model_tokenizer.vocab[
56 - self.tool_call_end_token]
53 + self.tool_call_start_token_id: int = self.model_tokenizer.vocab.get(
54 + self.tool_call_start_token, None)
55 + self.tool_call_end_token_id: int = self.model_tokenizer.vocab.get(
56 + self.tool_call_end_token, None)
57 if not self.tool_call_start_token_id or not self.tool_call_end_token_id:
58 raise RuntimeError(
59 "Hermes 2 Pro Tool parser could not locate tool call start/end "
vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py
+6 -1
@@ -61,8 +61,13 @@ class MistralToolParser(ToolParser):
61 self.streamed_args_for_tool: List[str] = [
62 ] # map what has been streamed for each tool so far to a list
63 self.bot_token = "[TOOL_CALLS]"
64 - self.bot_token_id = self.model_tokenizer.get_vocab()[self.bot_token]
64 + self.bot_token_id = self.model_tokenizer.get_vocab().get(
65 + self.bot_token, None)
66 self.tool_call_regex = re.compile(r"\[{.*?}\]", re.DOTALL)
67 + if not self.bot_token_id:
68 + raise RuntimeError(
69 + "Mistral Tool Parser could not locate the tool call token in "
70 + "the tokenizer!")
71
72 def extract_tool_calls(
73 self,