A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] IBM Granite 3.1 (#11307)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Dec 18, 2024 at 20:27 UTC
17ca964273464fad7e682380bab8288d4fac05c5
4 files changed
+27
-6
docs/source/models/supported_models.rst
+2
-2
@@ -194,8 +194,8 @@ Text Generation (``--task generate``)
194
-
195
- ✅︎
196
* - :code:`GraniteForCausalLM`
197
- - Granite 3.0, PowerLM
198
- - :code:`ibm-granite/granite-3.0-2b-base`, :code:`ibm-granite/granite-3.0-8b-instruct`, :code:`ibm/PowerLM-3b`, etc.
197
+ - Granite 3.0, Granite 3.1, PowerLM
198
+ - :code:`ibm-granite/granite-3.0-2b-base`, :code:`ibm-granite/granite-3.1-8b-instruct`, :code:`ibm/PowerLM-3b`, etc.
199
- ✅︎
200
- ✅︎
201
* - :code:`GraniteMoeForCausalLM`
docs/source/usage/tool_calling.md
+6
-1
@@ -170,6 +170,12 @@ Recommended flags: `--tool-call-parser granite --chat-template examples/tool_cha
170
171
`examples/tool_chat_template_granite.jinja`: this is a modified chat template from the original on Huggingface. Parallel function calls are supported.
172
173
+* `ibm-granite/granite-3.1-8b-instruct`
174
+
175
+Recommended flags: `--tool-call-parser granite`
176
+
177
+The chat template from Huggingface can be used directly. Parallel function calls are supported.
178
+
179
* `ibm-granite/granite-20b-functioncalling`
180
181
Recommended flags: `--tool-call-parser granite-20b-fc --chat-template examples/tool_chat_template_granite_20b_fc.jinja`
@@ -284,4 +290,3 @@ Then you can use this plugin in the command line like this.
290
--tool-call-parser example \
291
--chat-template <your chat template> \
292
```
287
-
tests/tool_use/utils.py
+9
-1
@@ -103,7 +103,7 @@ CONFIGS: Dict[str, ServerConfig] = {
103
"supports_rocm":
104
False,
105
},
106
- "granite8b": {
106
+ "granite-3.0-8b": {
107
"model":
108
"ibm-granite/granite-3.0-8b-instruct",
109
"arguments": [
@@ -111,6 +111,14 @@ CONFIGS: Dict[str, ServerConfig] = {
111
str(VLLM_PATH / "examples/tool_chat_template_granite.jinja")
112
],
113
},
114
+ "granite-3.1-8b": {
115
+ "model": "ibm-granite/granite-3.1-8b-instruct",
116
+ "arguments": [
117
+ "--tool-call-parser",
118
+ "granite",
119
+ ],
120
+ "supports_parallel": True,
121
+ },
122
"internlm": {
123
"model":
124
"internlm/internlm2_5-7b-chat",
vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py
+10
-2
@@ -35,13 +35,18 @@ class GraniteToolParser(ToolParser):
35
36
def __init__(self, tokenizer: AnyTokenizer):
37
super().__init__(tokenizer)
38
+ # for granite 3.0, the token `<|tool_call|>`
39
self.bot_token = "<|tool_call|>"
40
+ # for granite 3.1, the string `<tool_call>`
41
+ self.bot_string = "<tool_call>"
42
43
def extract_tool_calls(
44
self, model_output: str,
45
request: ChatCompletionRequest) -> ExtractedToolCallInformation:
43
- # remove whitespace and the BOT token if it exists
44
- stripped = model_output.strip().removeprefix(self.bot_token).lstrip()
46
+ stripped = model_output.strip()\
47
+ .removeprefix(self.bot_token)\
48
+ .removeprefix(self.bot_string)\
49
+ .lstrip()
50
if not stripped or stripped[0] != '[':
51
return ExtractedToolCallInformation(tools_called=False,
52
tool_calls=[],
@@ -91,6 +96,9 @@ class GraniteToolParser(ToolParser):
96
if current_text[start_idx:].startswith(self.bot_token):
97
start_idx = consume_space(start_idx + len(self.bot_token),
98
current_text)
99
+ if current_text[start_idx:].startswith(self.bot_string):
100
+ start_idx = consume_space(start_idx + len(self.bot_string),
101
+ current_text)
102
if not current_text or start_idx >= len(current_text)\
103
or current_text[start_idx] != '[':
104
return DeltaMessage(content=delta_text)