@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 17ca964273
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] IBM Granite 3.1 (#11307)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Dec 18, 2024 at 20:27 UTC 17ca964273464fad7e682380bab8288d4fac05c5
4 files changed +27 -6
docs/source/models/supported_models.rst
+2 -2
@@ -194,8 +194,8 @@ Text Generation (``--task generate``)
194 -
195 - ✅︎
196 * - :code:`GraniteForCausalLM`
197 - - Granite 3.0, PowerLM
198 - - :code:`ibm-granite/granite-3.0-2b-base`, :code:`ibm-granite/granite-3.0-8b-instruct`, :code:`ibm/PowerLM-3b`, etc.
197 + - Granite 3.0, Granite 3.1, PowerLM
198 + - :code:`ibm-granite/granite-3.0-2b-base`, :code:`ibm-granite/granite-3.1-8b-instruct`, :code:`ibm/PowerLM-3b`, etc.
199 - ✅︎
200 - ✅︎
201 * - :code:`GraniteMoeForCausalLM`
docs/source/usage/tool_calling.md
+6 -1
@@ -170,6 +170,12 @@ Recommended flags: `--tool-call-parser granite --chat-template examples/tool_cha
170
171 `examples/tool_chat_template_granite.jinja`: this is a modified chat template from the original on Huggingface. Parallel function calls are supported.
172
173 +* `ibm-granite/granite-3.1-8b-instruct`
174 +
175 +Recommended flags: `--tool-call-parser granite`
176 +
177 +The chat template from Huggingface can be used directly. Parallel function calls are supported.
178 +
179 * `ibm-granite/granite-20b-functioncalling`
180
181 Recommended flags: `--tool-call-parser granite-20b-fc --chat-template examples/tool_chat_template_granite_20b_fc.jinja`
@@ -284,4 +290,3 @@ Then you can use this plugin in the command line like this.
290 --tool-call-parser example \
291 --chat-template <your chat template> \
292 ```
287 -
tests/tool_use/utils.py
+9 -1
@@ -103,7 +103,7 @@ CONFIGS: Dict[str, ServerConfig] = {
103 "supports_rocm":
104 False,
105 },
106 - "granite8b": {
106 + "granite-3.0-8b": {
107 "model":
108 "ibm-granite/granite-3.0-8b-instruct",
109 "arguments": [
@@ -111,6 +111,14 @@ CONFIGS: Dict[str, ServerConfig] = {
111 str(VLLM_PATH / "examples/tool_chat_template_granite.jinja")
112 ],
113 },
114 + "granite-3.1-8b": {
115 + "model": "ibm-granite/granite-3.1-8b-instruct",
116 + "arguments": [
117 + "--tool-call-parser",
118 + "granite",
119 + ],
120 + "supports_parallel": True,
121 + },
122 "internlm": {
123 "model":
124 "internlm/internlm2_5-7b-chat",
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+10 -2
@@ -35,13 +35,18 @@ class GraniteToolParser(ToolParser):
35
36 def __init__(self, tokenizer: AnyTokenizer):
37 super().__init__(tokenizer)
38 + # for granite 3.0, the token `<|tool_call|>`
39 self.bot_token = "<|tool_call|>"
40 + # for granite 3.1, the string `<tool_call>`
41 + self.bot_string = "<tool_call>"
42
43 def extract_tool_calls(
44 self, model_output: str,
45 request: ChatCompletionRequest) -> ExtractedToolCallInformation:
43 - # remove whitespace and the BOT token if it exists
44 - stripped = model_output.strip().removeprefix(self.bot_token).lstrip()
46 + stripped = model_output.strip()\
47 + .removeprefix(self.bot_token)\
48 + .removeprefix(self.bot_string)\
49 + .lstrip()
50 if not stripped or stripped[0] != '[':
51 return ExtractedToolCallInformation(tools_called=False,
52 tool_calls=[],
@@ -91,6 +96,9 @@ class GraniteToolParser(ToolParser):
96 if current_text[start_idx:].startswith(self.bot_token):
97 start_idx = consume_space(start_idx + len(self.bot_token),
98 current_text)
99 + if current_text[start_idx:].startswith(self.bot_string):
100 + start_idx = consume_space(start_idx + len(self.bot_string),
101 + current_text)
102 if not current_text or start_idx >= len(current_text)\
103 or current_text[start_idx] != '[':
104 return DeltaMessage(content=delta_text)