[Rust Frontend] Replay roundtrip output grammars through XGrammar (#59143)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Bugen Zhao committed
Sep 30, 2026 at 15:29 UTC
87228cff01d7a56988e572c9a689444728f26568
17 files changed
+4486
-92
rust/src/chat/Cargo.toml
+1
@@ -46,6 +46,7 @@ clap.workspace = true
46
expect-test.workspace = true
47
paste.workspace = true
48
rmp-serde.workspace = true
49
+serde_json = { workspace = true, features = ["raw_value"] }
50
serial_test.workspace = true
51
tempfile.workspace = true
52
tokio.workspace = true
rust/src/chat/tests/grammar_replay.py
new
+113
@@ -0,0 +1,113 @@
1
+#!/usr/bin/env -S uv run --script
2
+# SPDX-License-Identifier: Apache-2.0
3
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4
+# /// script
5
+# requires-python = ">=3.11"
6
+# dependencies = [
7
+# "tiktoken>=0.6.0",
8
+# # xgrammar imports torch unconditionally; only the CPU build is needed.
9
+# "torch",
10
+# "transformers>=5.10.4",
11
+# # Keep in sync with requirements/common.txt.
12
+# "xgrammar==0.2.7",
13
+# ]
14
+#
15
+# [[tool.uv.index]]
16
+# name = "pytorch-cpu"
17
+# url = "https://download.pytorch.org/whl/cpu"
18
+# explicit = true
19
+#
20
+# [tool.uv.sources]
21
+# torch = { index = "pytorch-cpu" }
22
+# ///
23
+"""Replay the grammar cases exported by the roundtrip tests through XGrammar.
24
+
25
+Each file holds one model's cases, keyed by tool-choice variant. Every case is
26
+compiled against the model tokenizer the same way as
27
+`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then
28
+replayed from the first generated token: each token must be allowed by the
29
+bitmask and accepted by the matcher, and the final stop token must terminate it.
30
+"""
31
+
32
+import json
33
+import sys
34
+from functools import cache
35
+from pathlib import Path
36
+
37
+import xgrammar as xgr
38
+from transformers import AutoTokenizer
39
+
40
+CASES_DIR = Path(__file__).parent / "grammar_replay"
41
+
42
+
43
+@cache
44
+def load_tokenizer(model_id: str, vocab_size: int):
45
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
46
+ tokenizer_info = xgr.TokenizerInfo.from_huggingface(
47
+ tokenizer, vocab_size=vocab_size
48
+ )
49
+ return tokenizer, xgr.GrammarCompiler(tokenizer_info)
50
+
51
+
52
+def replay(model: dict, case: dict) -> str:
53
+ """Check one case of a model and return a summary of what was checked."""
54
+ tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"])
55
+ generation = case["generation_token_ids"]
56
+
57
+ # The case token IDs come from the Rust tokenizer; the runner must load the
58
+ # same vocabulary for the grammar to mean the same thing.
59
+ encoded = tokenizer.encode(case["completion"], add_special_tokens=False)
60
+ if encoded != generation[:-1]:
61
+ raise AssertionError(
62
+ "tokenizer mismatch: the completion encodes to "
63
+ f"{encoded}, but the case records {generation[:-1]}"
64
+ )
65
+
66
+ grammar = case["grammar"]
67
+ structural_tag = {"type": "structural_tag", "format": grammar["format"]}
68
+ compiled = compiler.compile_structural_tag(json.dumps(structural_tag))
69
+ if grammar["coverage"] != "from_token_zero":
70
+ return f"compiled only ({grammar['coverage']} is gated by the engine)"
71
+
72
+ matcher = xgr.GrammarMatcher(
73
+ compiled, override_stop_tokens=model["all_stop_token_ids"]
74
+ )
75
+ bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"])
76
+ for index, token_id in enumerate(generation):
77
+ matcher.fill_next_token_bitmask(bitmask)
78
+ allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1
79
+ if not allowed or not matcher.accept_token(token_id):
80
+ prefix = tokenizer.decode(generation[:index])
81
+ raise AssertionError(
82
+ f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) "
83
+ f"rejected after {prefix!r}"
84
+ )
85
+ if not matcher.is_terminated():
86
+ raise AssertionError("matcher not terminated after the stop token")
87
+ return f"replayed {len(generation)} tokens from token zero"
88
+
89
+
90
+def main() -> int:
91
+ paths = sorted(CASES_DIR.glob("*.json"))
92
+ if not paths:
93
+ print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr)
94
+ return 1
95
+
96
+ failures = 0
97
+ for path in paths:
98
+ model = json.loads(path.read_text())
99
+ for name, case in model["cases"].items():
100
+ try:
101
+ summary = replay(model, case)
102
+ except Exception as error:
103
+ failures += 1
104
+ coverage = case["grammar"]["coverage"]
105
+ print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr)
106
+ else:
107
+ print(f"PASS {path.name} {name}: {summary}")
108
+
109
+ return 1 if failures else 0
110
+
111
+
112
+if __name__ == "__main__":
113
+ sys.exit(main())
rust/src/chat/tests/grammar_replay.rs
new
+24
@@ -0,0 +1,24 @@
1
+// SPDX-License-Identifier: Apache-2.0
2
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+//! Replays the grammar cases exported by the roundtrip tests through the real
5
+//! XGrammar, so that parser-built output grammars are checked against the
6
+//! engine's compiler and matcher.
7
+
8
+use std::path::PathBuf;
9
+use std::process::Command;
10
+
11
+#[test]
12
+fn xgrammar_accepts_roundtrip_generations() {
13
+ let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay.py");
14
+ let output = Command::new(&script)
15
+ .output()
16
+ .unwrap_or_else(|error| panic!("failed to execute {:?}: {error}", script));
17
+ assert!(
18
+ output.status.success(),
19
+ "grammar replay failed: status={:?}\nstdout:\n{}\nstderr:\n{}",
20
+ output.status.code(),
21
+ String::from_utf8_lossy(&output.stdout),
22
+ String::from_utf8_lossy(&output.stderr),
23
+ );
24
+}
rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json
new
+274
@@ -0,0 +1,274 @@
1
+{
2
+ "model_id": "deepseek-ai/DeepSeek-V3.2-Exp",
3
+ "vocab_size": 129280,
4
+ "all_stop_token_ids": [1],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<｜DSML｜function_calls>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<｜DSML｜function_calls>\n",
31
+ "content": {
32
+ "type": "tags_with_separator",
33
+ "tags": [
34
+ {
35
+ "type": "tag",
36
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
37
+ "content": {
38
+ "type": "json_schema",
39
+ "json_schema": {
40
+ "type": "object",
41
+ "properties": {
42
+ "location": {
43
+ "type": "string"
44
+ }
45
+ },
46
+ "required": [
47
+ "location"
48
+ ]
49
+ },
50
+ "style": "deepseek_xml",
51
+ "any_order": false,
52
+ "max_whitespace_cnt": null,
53
+ "excludes": []
54
+ },
55
+ "end": "</｜DSML｜invoke>\n"
56
+ },
57
+ {
58
+ "type": "tag",
59
+ "begin": "<｜DSML｜invoke name=\"add\">\n",
60
+ "content": {
61
+ "type": "json_schema",
62
+ "json_schema": {
63
+ "type": "object",
64
+ "properties": {
65
+ "y": {
66
+ "type": "number"
67
+ },
68
+ "x": {
69
+ "type": "number"
70
+ },
71
+ "items": {
72
+ "type": "array",
73
+ "items": {
74
+ "type": "string"
75
+ }
76
+ }
77
+ },
78
+ "required": [
79
+ "y",
80
+ "x",
81
+ "items"
82
+ ]
83
+ },
84
+ "style": "deepseek_xml",
85
+ "any_order": false,
86
+ "max_whitespace_cnt": null,
87
+ "excludes": []
88
+ },
89
+ "end": "</｜DSML｜invoke>\n"
90
+ }
91
+ ],
92
+ "separator": "",
93
+ "at_least_one": true,
94
+ "stop_after_first": false
95
+ },
96
+ "end": "</｜DSML｜function_calls>"
97
+ }
98
+ ],
99
+ "at_least_one": false,
100
+ "stop_after_first": false,
101
+ "excludes": [
102
+ "<think>",
103
+ "</think>"
104
+ ]
105
+ }
106
+ ]
107
+ },
108
+ "coverage": "from_token_zero"
109
+ },
110
+ "completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0</｜DSML｜parameter>\n<｜DSML｜parameter name=\"x\" string=\"false\">2</｜DSML｜parameter>\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜function_calls>",
111
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
112
+ },
113
+ "required": {
114
+ "grammar": {
115
+ "format": {
116
+ "type": "sequence",
117
+ "elements": [
118
+ {
119
+ "type": "tag",
120
+ "begin": "",
121
+ "content": {
122
+ "type": "any_text",
123
+ "excludes": [],
124
+ "max_tokens": null,
125
+ "max_chars": null
126
+ },
127
+ "end": "</think>"
128
+ },
129
+ {
130
+ "type": "sequence",
131
+ "elements": [
132
+ {
133
+ "type": "const_string",
134
+ "value": "\n\n<｜DSML｜function_calls>\n"
135
+ },
136
+ {
137
+ "type": "tags_with_separator",
138
+ "tags": [
139
+ {
140
+ "type": "tag",
141
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
142
+ "content": {
143
+ "type": "json_schema",
144
+ "json_schema": {
145
+ "type": "object",
146
+ "properties": {
147
+ "location": {
148
+ "type": "string"
149
+ }
150
+ },
151
+ "required": [
152
+ "location"
153
+ ]
154
+ },
155
+ "style": "deepseek_xml",
156
+ "any_order": false,
157
+ "max_whitespace_cnt": null,
158
+ "excludes": []
159
+ },
160
+ "end": "</｜DSML｜invoke>\n"
161
+ },
162
+ {
163
+ "type": "tag",
164
+ "begin": "<｜DSML｜invoke name=\"add\">\n",
165
+ "content": {
166
+ "type": "json_schema",
167
+ "json_schema": {
168
+ "type": "object",
169
+ "properties": {
170
+ "y": {
171
+ "type": "number"
172
+ },
173
+ "x": {
174
+ "type": "number"
175
+ },
176
+ "items": {
177
+ "type": "array",
178
+ "items": {
179
+ "type": "string"
180
+ }
181
+ }
182
+ },
183
+ "required": [
184
+ "y",
185
+ "x",
186
+ "items"
187
+ ]
188
+ },
189
+ "style": "deepseek_xml",
190
+ "any_order": false,
191
+ "max_whitespace_cnt": null,
192
+ "excludes": []
193
+ },
194
+ "end": "</｜DSML｜invoke>\n"
195
+ }
196
+ ],
197
+ "separator": "",
198
+ "at_least_one": true,
199
+ "stop_after_first": false
200
+ },
201
+ {
202
+ "type": "const_string",
203
+ "value": "</｜DSML｜function_calls>"
204
+ }
205
+ ]
206
+ }
207
+ ]
208
+ },
209
+ "coverage": "from_token_zero"
210
+ },
211
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0</｜DSML｜parameter>\n<｜DSML｜parameter name=\"x\" string=\"false\">2</｜DSML｜parameter>\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜function_calls>",
212
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
213
+ },
214
+ "named": {
215
+ "grammar": {
216
+ "format": {
217
+ "type": "sequence",
218
+ "elements": [
219
+ {
220
+ "type": "tag",
221
+ "begin": "",
222
+ "content": {
223
+ "type": "any_text",
224
+ "excludes": [],
225
+ "max_tokens": null,
226
+ "max_chars": null
227
+ },
228
+ "end": "</think>"
229
+ },
230
+ {
231
+ "type": "sequence",
232
+ "elements": [
233
+ {
234
+ "type": "const_string",
235
+ "value": "\n\n<｜DSML｜function_calls>\n"
236
+ },
237
+ {
238
+ "type": "tag",
239
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
240
+ "content": {
241
+ "type": "json_schema",
242
+ "json_schema": {
243
+ "type": "object",
244
+ "properties": {
245
+ "location": {
246
+ "type": "string"
247
+ }
248
+ },
249
+ "required": [
250
+ "location"
251
+ ]
252
+ },
253
+ "style": "deepseek_xml",
254
+ "any_order": false,
255
+ "max_whitespace_cnt": null,
256
+ "excludes": []
257
+ },
258
+ "end": "</｜DSML｜invoke>\n"
259
+ },
260
+ {
261
+ "type": "const_string",
262
+ "value": "</｜DSML｜function_calls>"
263
+ }
264
+ ]
265
+ }
266
+ ]
267
+ },
268
+ "coverage": "from_token_zero"
269
+ },
270
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜function_calls>",
271
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1]
272
+ }
273
+ }
274
+}
rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json
new
+274
@@ -0,0 +1,274 @@
1
+{
2
+ "model_id": "deepseek-ai/DeepSeek-V4-Flash",
3
+ "vocab_size": 129280,
4
+ "all_stop_token_ids": [1],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<｜DSML｜tool_calls>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<｜DSML｜tool_calls>\n",
31
+ "content": {
32
+ "type": "tags_with_separator",
33
+ "tags": [
34
+ {
35
+ "type": "tag",
36
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
37
+ "content": {
38
+ "type": "json_schema",
39
+ "json_schema": {
40
+ "type": "object",
41
+ "properties": {
42
+ "location": {
43
+ "type": "string"
44
+ }
45
+ },
46
+ "required": [
47
+ "location"
48
+ ]
49
+ },
50
+ "style": "deepseek_xml",
51
+ "any_order": false,
52
+ "max_whitespace_cnt": null,
53
+ "excludes": []
54
+ },
55
+ "end": "</｜DSML｜invoke>\n"
56
+ },
57
+ {
58
+ "type": "tag",
59
+ "begin": "<｜DSML｜invoke name=\"add\">\n",
60
+ "content": {
61
+ "type": "json_schema",
62
+ "json_schema": {
63
+ "type": "object",
64
+ "properties": {
65
+ "y": {
66
+ "type": "number"
67
+ },
68
+ "x": {
69
+ "type": "number"
70
+ },
71
+ "items": {
72
+ "type": "array",
73
+ "items": {
74
+ "type": "string"
75
+ }
76
+ }
77
+ },
78
+ "required": [
79
+ "y",
80
+ "x",
81
+ "items"
82
+ ]
83
+ },
84
+ "style": "deepseek_xml",
85
+ "any_order": false,
86
+ "max_whitespace_cnt": null,
87
+ "excludes": []
88
+ },
89
+ "end": "</｜DSML｜invoke>\n"
90
+ }
91
+ ],
92
+ "separator": "",
93
+ "at_least_one": true,
94
+ "stop_after_first": false
95
+ },
96
+ "end": "</｜DSML｜tool_calls>"
97
+ }
98
+ ],
99
+ "at_least_one": false,
100
+ "stop_after_first": false,
101
+ "excludes": [
102
+ "<think>",
103
+ "</think>"
104
+ ]
105
+ }
106
+ ]
107
+ },
108
+ "coverage": "from_token_zero"
109
+ },
110
+ "completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0</｜DSML｜parameter>\n<｜DSML｜parameter name=\"x\" string=\"false\">2</｜DSML｜parameter>\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>",
111
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
112
+ },
113
+ "required": {
114
+ "grammar": {
115
+ "format": {
116
+ "type": "sequence",
117
+ "elements": [
118
+ {
119
+ "type": "tag",
120
+ "begin": "",
121
+ "content": {
122
+ "type": "any_text",
123
+ "excludes": [],
124
+ "max_tokens": null,
125
+ "max_chars": null
126
+ },
127
+ "end": "</think>"
128
+ },
129
+ {
130
+ "type": "sequence",
131
+ "elements": [
132
+ {
133
+ "type": "const_string",
134
+ "value": "\n\n<｜DSML｜tool_calls>\n"
135
+ },
136
+ {
137
+ "type": "tags_with_separator",
138
+ "tags": [
139
+ {
140
+ "type": "tag",
141
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
142
+ "content": {
143
+ "type": "json_schema",
144
+ "json_schema": {
145
+ "type": "object",
146
+ "properties": {
147
+ "location": {
148
+ "type": "string"
149
+ }
150
+ },
151
+ "required": [
152
+ "location"
153
+ ]
154
+ },
155
+ "style": "deepseek_xml",
156
+ "any_order": false,
157
+ "max_whitespace_cnt": null,
158
+ "excludes": []
159
+ },
160
+ "end": "</｜DSML｜invoke>\n"
161
+ },
162
+ {
163
+ "type": "tag",
164
+ "begin": "<｜DSML｜invoke name=\"add\">\n",
165
+ "content": {
166
+ "type": "json_schema",
167
+ "json_schema": {
168
+ "type": "object",
169
+ "properties": {
170
+ "y": {
171
+ "type": "number"
172
+ },
173
+ "x": {
174
+ "type": "number"
175
+ },
176
+ "items": {
177
+ "type": "array",
178
+ "items": {
179
+ "type": "string"
180
+ }
181
+ }
182
+ },
183
+ "required": [
184
+ "y",
185
+ "x",
186
+ "items"
187
+ ]
188
+ },
189
+ "style": "deepseek_xml",
190
+ "any_order": false,
191
+ "max_whitespace_cnt": null,
192
+ "excludes": []
193
+ },
194
+ "end": "</｜DSML｜invoke>\n"
195
+ }
196
+ ],
197
+ "separator": "",
198
+ "at_least_one": true,
199
+ "stop_after_first": false
200
+ },
201
+ {
202
+ "type": "const_string",
203
+ "value": "</｜DSML｜tool_calls>"
204
+ }
205
+ ]
206
+ }
207
+ ]
208
+ },
209
+ "coverage": "from_token_zero"
210
+ },
211
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0</｜DSML｜parameter>\n<｜DSML｜parameter name=\"x\" string=\"false\">2</｜DSML｜parameter>\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>",
212
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
213
+ },
214
+ "named": {
215
+ "grammar": {
216
+ "format": {
217
+ "type": "sequence",
218
+ "elements": [
219
+ {
220
+ "type": "tag",
221
+ "begin": "",
222
+ "content": {
223
+ "type": "any_text",
224
+ "excludes": [],
225
+ "max_tokens": null,
226
+ "max_chars": null
227
+ },
228
+ "end": "</think>"
229
+ },
230
+ {
231
+ "type": "sequence",
232
+ "elements": [
233
+ {
234
+ "type": "const_string",
235
+ "value": "\n\n<｜DSML｜tool_calls>\n"
236
+ },
237
+ {
238
+ "type": "tag",
239
+ "begin": "<｜DSML｜invoke name=\"get_weather\">\n",
240
+ "content": {
241
+ "type": "json_schema",
242
+ "json_schema": {
243
+ "type": "object",
244
+ "properties": {
245
+ "location": {
246
+ "type": "string"
247
+ }
248
+ },
249
+ "required": [
250
+ "location"
251
+ ]
252
+ },
253
+ "style": "deepseek_xml",
254
+ "any_order": false,
255
+ "max_whitespace_cnt": null,
256
+ "excludes": []
257
+ },
258
+ "end": "</｜DSML｜invoke>\n"
259
+ },
260
+ {
261
+ "type": "const_string",
262
+ "value": "</｜DSML｜tool_calls>"
263
+ }
264
+ ]
265
+ }
266
+ ]
267
+ },
268
+ "coverage": "from_token_zero"
269
+ },
270
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜parameter>\n</｜DSML｜invoke>\n</｜DSML｜tool_calls>",
271
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1]
272
+ }
273
+ }
274
+}
rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json
new
+274
@@ -0,0 +1,274 @@
1
+{
2
+ "model_id": "deepseek-ai/DeepSeek-V4.1-Flash",
3
+ "vocab_size": 129280,
4
+ "all_stop_token_ids": [1],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<｜DSML｜ calls>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<｜DSML｜ calls>\n",
31
+ "content": {
32
+ "type": "tags_with_separator",
33
+ "tags": [
34
+ {
35
+ "type": "tag",
36
+ "begin": "<｜DSML｜ invoke name=\"get_weather\">\n",
37
+ "content": {
38
+ "type": "json_schema",
39
+ "json_schema": {
40
+ "type": "object",
41
+ "properties": {
42
+ "location": {
43
+ "type": "string"
44
+ }
45
+ },
46
+ "required": [
47
+ "location"
48
+ ]
49
+ },
50
+ "style": "deepseek_v4_1_xml",
51
+ "any_order": false,
52
+ "max_whitespace_cnt": null,
53
+ "excludes": []
54
+ },
55
+ "end": "</｜DSML｜ invoke>\n"
56
+ },
57
+ {
58
+ "type": "tag",
59
+ "begin": "<｜DSML｜ invoke name=\"add\">\n",
60
+ "content": {
61
+ "type": "json_schema",
62
+ "json_schema": {
63
+ "type": "object",
64
+ "properties": {
65
+ "y": {
66
+ "type": "number"
67
+ },
68
+ "x": {
69
+ "type": "number"
70
+ },
71
+ "items": {
72
+ "type": "array",
73
+ "items": {
74
+ "type": "string"
75
+ }
76
+ }
77
+ },
78
+ "required": [
79
+ "y",
80
+ "x",
81
+ "items"
82
+ ]
83
+ },
84
+ "style": "deepseek_v4_1_xml",
85
+ "any_order": false,
86
+ "max_whitespace_cnt": null,
87
+ "excludes": []
88
+ },
89
+ "end": "</｜DSML｜ invoke>\n"
90
+ }
91
+ ],
92
+ "separator": "",
93
+ "at_least_one": true,
94
+ "stop_after_first": false
95
+ },
96
+ "end": "</｜DSML｜ calls>"
97
+ }
98
+ ],
99
+ "at_least_one": false,
100
+ "stop_after_first": false,
101
+ "excludes": [
102
+ "<think>",
103
+ "</think>"
104
+ ]
105
+ }
106
+ ]
107
+ },
108
+ "coverage": "from_token_zero"
109
+ },
110
+ "completion": "Need call the weather and add tools.</think>I will call the tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0</｜DSML｜ parameter>\n<｜DSML｜ parameter name=\"x\" string=\"false\">2</｜DSML｜ parameter>\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>",
111
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
112
+ },
113
+ "required": {
114
+ "grammar": {
115
+ "format": {
116
+ "type": "sequence",
117
+ "elements": [
118
+ {
119
+ "type": "tag",
120
+ "begin": "",
121
+ "content": {
122
+ "type": "any_text",
123
+ "excludes": [],
124
+ "max_tokens": null,
125
+ "max_chars": null
126
+ },
127
+ "end": "</think>"
128
+ },
129
+ {
130
+ "type": "sequence",
131
+ "elements": [
132
+ {
133
+ "type": "const_string",
134
+ "value": "\n\n<｜DSML｜ calls>\n"
135
+ },
136
+ {
137
+ "type": "tags_with_separator",
138
+ "tags": [
139
+ {
140
+ "type": "tag",
141
+ "begin": "<｜DSML｜ invoke name=\"get_weather\">\n",
142
+ "content": {
143
+ "type": "json_schema",
144
+ "json_schema": {
145
+ "type": "object",
146
+ "properties": {
147
+ "location": {
148
+ "type": "string"
149
+ }
150
+ },
151
+ "required": [
152
+ "location"
153
+ ]
154
+ },
155
+ "style": "deepseek_v4_1_xml",
156
+ "any_order": false,
157
+ "max_whitespace_cnt": null,
158
+ "excludes": []
159
+ },
160
+ "end": "</｜DSML｜ invoke>\n"
161
+ },
162
+ {
163
+ "type": "tag",
164
+ "begin": "<｜DSML｜ invoke name=\"add\">\n",
165
+ "content": {
166
+ "type": "json_schema",
167
+ "json_schema": {
168
+ "type": "object",
169
+ "properties": {
170
+ "y": {
171
+ "type": "number"
172
+ },
173
+ "x": {
174
+ "type": "number"
175
+ },
176
+ "items": {
177
+ "type": "array",
178
+ "items": {
179
+ "type": "string"
180
+ }
181
+ }
182
+ },
183
+ "required": [
184
+ "y",
185
+ "x",
186
+ "items"
187
+ ]
188
+ },
189
+ "style": "deepseek_v4_1_xml",
190
+ "any_order": false,
191
+ "max_whitespace_cnt": null,
192
+ "excludes": []
193
+ },
194
+ "end": "</｜DSML｜ invoke>\n"
195
+ }
196
+ ],
197
+ "separator": "",
198
+ "at_least_one": true,
199
+ "stop_after_first": false
200
+ },
201
+ {
202
+ "type": "const_string",
203
+ "value": "</｜DSML｜ calls>"
204
+ }
205
+ ]
206
+ }
207
+ ]
208
+ },
209
+ "coverage": "from_token_zero"
210
+ },
211
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0</｜DSML｜ parameter>\n<｜DSML｜ parameter name=\"x\" string=\"false\">2</｜DSML｜ parameter>\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>",
212
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
213
+ },
214
+ "named": {
215
+ "grammar": {
216
+ "format": {
217
+ "type": "sequence",
218
+ "elements": [
219
+ {
220
+ "type": "tag",
221
+ "begin": "",
222
+ "content": {
223
+ "type": "any_text",
224
+ "excludes": [],
225
+ "max_tokens": null,
226
+ "max_chars": null
227
+ },
228
+ "end": "</think>"
229
+ },
230
+ {
231
+ "type": "sequence",
232
+ "elements": [
233
+ {
234
+ "type": "const_string",
235
+ "value": "\n\n<｜DSML｜ calls>\n"
236
+ },
237
+ {
238
+ "type": "tag",
239
+ "begin": "<｜DSML｜ invoke name=\"get_weather\">\n",
240
+ "content": {
241
+ "type": "json_schema",
242
+ "json_schema": {
243
+ "type": "object",
244
+ "properties": {
245
+ "location": {
246
+ "type": "string"
247
+ }
248
+ },
249
+ "required": [
250
+ "location"
251
+ ]
252
+ },
253
+ "style": "deepseek_v4_1_xml",
254
+ "any_order": false,
255
+ "max_whitespace_cnt": null,
256
+ "excludes": []
257
+ },
258
+ "end": "</｜DSML｜ invoke>\n"
259
+ },
260
+ {
261
+ "type": "const_string",
262
+ "value": "</｜DSML｜ calls>"
263
+ }
264
+ ]
265
+ }
266
+ ]
267
+ },
268
+ "coverage": "from_token_zero"
269
+ },
270
+ "completion": "Need call the weather and add tools.</think>\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>",
271
+ "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1]
272
+ }
273
+ }
274
+}
rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json
new
+251
@@ -0,0 +1,251 @@
1
+{
2
+ "model_id": "zai-org/GLM-4.7-Flash",
3
+ "vocab_size": 154880,
4
+ "all_stop_token_ids": [154820, 154827, 154829],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<tool_call>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<tool_call>get_weather",
31
+ "content": {
32
+ "type": "json_schema",
33
+ "json_schema": {
34
+ "type": "object",
35
+ "properties": {
36
+ "location": {
37
+ "type": "string"
38
+ }
39
+ },
40
+ "required": [
41
+ "location"
42
+ ]
43
+ },
44
+ "style": "glm_xml",
45
+ "any_order": false,
46
+ "max_whitespace_cnt": null,
47
+ "excludes": []
48
+ },
49
+ "end": "</tool_call>"
50
+ },
51
+ {
52
+ "type": "tag",
53
+ "begin": "<tool_call>add",
54
+ "content": {
55
+ "type": "json_schema",
56
+ "json_schema": {
57
+ "type": "object",
58
+ "properties": {
59
+ "y": {
60
+ "type": "number"
61
+ },
62
+ "x": {
63
+ "type": "number"
64
+ },
65
+ "items": {
66
+ "type": "array",
67
+ "items": {
68
+ "type": "string"
69
+ }
70
+ }
71
+ },
72
+ "required": [
73
+ "y",
74
+ "x",
75
+ "items"
76
+ ]
77
+ },
78
+ "style": "glm_xml",
79
+ "any_order": false,
80
+ "max_whitespace_cnt": null,
81
+ "excludes": []
82
+ },
83
+ "end": "</tool_call>"
84
+ }
85
+ ],
86
+ "at_least_one": false,
87
+ "stop_after_first": false,
88
+ "excludes": [
89
+ "<think>",
90
+ "</think>",
91
+ "</tool_call>",
92
+ "<arg_key>",
93
+ "</arg_key>",
94
+ "<arg_value>",
95
+ "</arg_value>"
96
+ ]
97
+ }
98
+ ]
99
+ },
100
+ "coverage": "from_token_zero"
101
+ },
102
+ "completion": "Need call the weather and add tools.</think>I will call the tools.<tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
103
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
104
+ },
105
+ "required": {
106
+ "grammar": {
107
+ "format": {
108
+ "type": "sequence",
109
+ "elements": [
110
+ {
111
+ "type": "tag",
112
+ "begin": "",
113
+ "content": {
114
+ "type": "any_text",
115
+ "excludes": [],
116
+ "max_tokens": null,
117
+ "max_chars": null
118
+ },
119
+ "end": "</think>"
120
+ },
121
+ {
122
+ "type": "triggered_tags",
123
+ "triggers": [
124
+ "<tool_call>"
125
+ ],
126
+ "tags": [
127
+ {
128
+ "type": "tag",
129
+ "begin": "<tool_call>get_weather",
130
+ "content": {
131
+ "type": "json_schema",
132
+ "json_schema": {
133
+ "type": "object",
134
+ "properties": {
135
+ "location": {
136
+ "type": "string"
137
+ }
138
+ },
139
+ "required": [
140
+ "location"
141
+ ]
142
+ },
143
+ "style": "glm_xml",
144
+ "any_order": false,
145
+ "max_whitespace_cnt": null,
146
+ "excludes": []
147
+ },
148
+ "end": "</tool_call>"
149
+ },
150
+ {
151
+ "type": "tag",
152
+ "begin": "<tool_call>add",
153
+ "content": {
154
+ "type": "json_schema",
155
+ "json_schema": {
156
+ "type": "object",
157
+ "properties": {
158
+ "y": {
159
+ "type": "number"
160
+ },
161
+ "x": {
162
+ "type": "number"
163
+ },
164
+ "items": {
165
+ "type": "array",
166
+ "items": {
167
+ "type": "string"
168
+ }
169
+ }
170
+ },
171
+ "required": [
172
+ "y",
173
+ "x",
174
+ "items"
175
+ ]
176
+ },
177
+ "style": "glm_xml",
178
+ "any_order": false,
179
+ "max_whitespace_cnt": null,
180
+ "excludes": []
181
+ },
182
+ "end": "</tool_call>"
183
+ }
184
+ ],
185
+ "at_least_one": true,
186
+ "stop_after_first": false,
187
+ "excludes": [
188
+ "<think>",
189
+ "</think>",
190
+ "</tool_call>",
191
+ "<arg_key>",
192
+ "</arg_key>",
193
+ "<arg_value>",
194
+ "</arg_value>"
195
+ ]
196
+ }
197
+ ]
198
+ },
199
+ "coverage": "from_token_zero"
200
+ },
201
+ "completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
202
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
203
+ },
204
+ "named": {
205
+ "grammar": {
206
+ "format": {
207
+ "type": "sequence",
208
+ "elements": [
209
+ {
210
+ "type": "tag",
211
+ "begin": "",
212
+ "content": {
213
+ "type": "any_text",
214
+ "excludes": [],
215
+ "max_tokens": null,
216
+ "max_chars": null
217
+ },
218
+ "end": "</think>"
219
+ },
220
+ {
221
+ "type": "tag",
222
+ "begin": "<tool_call>get_weather",
223
+ "content": {
224
+ "type": "json_schema",
225
+ "json_schema": {
226
+ "type": "object",
227
+ "properties": {
228
+ "location": {
229
+ "type": "string"
230
+ }
231
+ },
232
+ "required": [
233
+ "location"
234
+ ]
235
+ },
236
+ "style": "glm_xml",
237
+ "any_order": false,
238
+ "max_whitespace_cnt": null,
239
+ "excludes": []
240
+ },
241
+ "end": "</tool_call>"
242
+ }
243
+ ]
244
+ },
245
+ "coverage": "from_token_zero"
246
+ },
247
+ "completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call>",
248
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820]
249
+ }
250
+ }
251
+}
rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json
new
+251
@@ -0,0 +1,251 @@
1
+{
2
+ "model_id": "zai-org/GLM-5.2-FP8",
3
+ "vocab_size": 154880,
4
+ "all_stop_token_ids": [154820, 154827, 154829],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<tool_call>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<tool_call>get_weather",
31
+ "content": {
32
+ "type": "json_schema",
33
+ "json_schema": {
34
+ "type": "object",
35
+ "properties": {
36
+ "location": {
37
+ "type": "string"
38
+ }
39
+ },
40
+ "required": [
41
+ "location"
42
+ ]
43
+ },
44
+ "style": "glm_xml",
45
+ "any_order": false,
46
+ "max_whitespace_cnt": null,
47
+ "excludes": []
48
+ },
49
+ "end": "</tool_call>"
50
+ },
51
+ {
52
+ "type": "tag",
53
+ "begin": "<tool_call>add",
54
+ "content": {
55
+ "type": "json_schema",
56
+ "json_schema": {
57
+ "type": "object",
58
+ "properties": {
59
+ "y": {
60
+ "type": "number"
61
+ },
62
+ "x": {
63
+ "type": "number"
64
+ },
65
+ "items": {
66
+ "type": "array",
67
+ "items": {
68
+ "type": "string"
69
+ }
70
+ }
71
+ },
72
+ "required": [
73
+ "y",
74
+ "x",
75
+ "items"
76
+ ]
77
+ },
78
+ "style": "glm_xml",
79
+ "any_order": false,
80
+ "max_whitespace_cnt": null,
81
+ "excludes": []
82
+ },
83
+ "end": "</tool_call>"
84
+ }
85
+ ],
86
+ "at_least_one": false,
87
+ "stop_after_first": false,
88
+ "excludes": [
89
+ "<think>",
90
+ "</think>",
91
+ "</tool_call>",
92
+ "<arg_key>",
93
+ "</arg_key>",
94
+ "<arg_value>",
95
+ "</arg_value>"
96
+ ]
97
+ }
98
+ ]
99
+ },
100
+ "coverage": "from_token_zero"
101
+ },
102
+ "completion": "Need call the weather and add tools.</think>I will call the tools.<tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
103
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
104
+ },
105
+ "required": {
106
+ "grammar": {
107
+ "format": {
108
+ "type": "sequence",
109
+ "elements": [
110
+ {
111
+ "type": "tag",
112
+ "begin": "",
113
+ "content": {
114
+ "type": "any_text",
115
+ "excludes": [],
116
+ "max_tokens": null,
117
+ "max_chars": null
118
+ },
119
+ "end": "</think>"
120
+ },
121
+ {
122
+ "type": "triggered_tags",
123
+ "triggers": [
124
+ "<tool_call>"
125
+ ],
126
+ "tags": [
127
+ {
128
+ "type": "tag",
129
+ "begin": "<tool_call>get_weather",
130
+ "content": {
131
+ "type": "json_schema",
132
+ "json_schema": {
133
+ "type": "object",
134
+ "properties": {
135
+ "location": {
136
+ "type": "string"
137
+ }
138
+ },
139
+ "required": [
140
+ "location"
141
+ ]
142
+ },
143
+ "style": "glm_xml",
144
+ "any_order": false,
145
+ "max_whitespace_cnt": null,
146
+ "excludes": []
147
+ },
148
+ "end": "</tool_call>"
149
+ },
150
+ {
151
+ "type": "tag",
152
+ "begin": "<tool_call>add",
153
+ "content": {
154
+ "type": "json_schema",
155
+ "json_schema": {
156
+ "type": "object",
157
+ "properties": {
158
+ "y": {
159
+ "type": "number"
160
+ },
161
+ "x": {
162
+ "type": "number"
163
+ },
164
+ "items": {
165
+ "type": "array",
166
+ "items": {
167
+ "type": "string"
168
+ }
169
+ }
170
+ },
171
+ "required": [
172
+ "y",
173
+ "x",
174
+ "items"
175
+ ]
176
+ },
177
+ "style": "glm_xml",
178
+ "any_order": false,
179
+ "max_whitespace_cnt": null,
180
+ "excludes": []
181
+ },
182
+ "end": "</tool_call>"
183
+ }
184
+ ],
185
+ "at_least_one": true,
186
+ "stop_after_first": false,
187
+ "excludes": [
188
+ "<think>",
189
+ "</think>",
190
+ "</tool_call>",
191
+ "<arg_key>",
192
+ "</arg_key>",
193
+ "<arg_value>",
194
+ "</arg_value>"
195
+ ]
196
+ }
197
+ ]
198
+ },
199
+ "coverage": "from_token_zero"
200
+ },
201
+ "completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call><tool_call>add<arg_key>y</arg_key><arg_value>1.0</arg_value><arg_key>x</arg_key><arg_value>2</arg_value><arg_key>items</arg_key><arg_value>[\"left\", \"right\"]</arg_value></tool_call>",
202
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820]
203
+ },
204
+ "named": {
205
+ "grammar": {
206
+ "format": {
207
+ "type": "sequence",
208
+ "elements": [
209
+ {
210
+ "type": "tag",
211
+ "begin": "",
212
+ "content": {
213
+ "type": "any_text",
214
+ "excludes": [],
215
+ "max_tokens": null,
216
+ "max_chars": null
217
+ },
218
+ "end": "</think>"
219
+ },
220
+ {
221
+ "type": "tag",
222
+ "begin": "<tool_call>get_weather",
223
+ "content": {
224
+ "type": "json_schema",
225
+ "json_schema": {
226
+ "type": "object",
227
+ "properties": {
228
+ "location": {
229
+ "type": "string"
230
+ }
231
+ },
232
+ "required": [
233
+ "location"
234
+ ]
235
+ },
236
+ "style": "glm_xml",
237
+ "any_order": false,
238
+ "max_whitespace_cnt": null,
239
+ "excludes": []
240
+ },
241
+ "end": "</tool_call>"
242
+ }
243
+ ]
244
+ },
245
+ "coverage": "from_token_zero"
246
+ },
247
+ "completion": "Need call the weather and add tools.</think><tool_call>get_weather<arg_key>location</arg_key><arg_value>Shanghai</arg_value></tool_call>",
248
+ "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820]
249
+ }
250
+ }
251
+}
rust/src/chat/tests/grammar_replay/Hy3.json
new
+625
@@ -0,0 +1,625 @@
1
+{
2
+ "model_id": "tencent/Hy3",
3
+ "vocab_size": 120832,
4
+ "all_stop_token_ids": [120025],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think:opensource>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<tool_calls:opensource>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<tool_calls:opensource>\n",
31
+ "content": {
32
+ "type": "tags_with_separator",
33
+ "tags": [
34
+ {
35
+ "type": "tag",
36
+ "begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
37
+ "content": {
38
+ "type": "sequence",
39
+ "elements": [
40
+ {
41
+ "type": "sequence",
42
+ "elements": [
43
+ {
44
+ "type": "const_string",
45
+ "value": "<arg_key:opensource>"
46
+ },
47
+ {
48
+ "type": "const_string",
49
+ "value": "location"
50
+ },
51
+ {
52
+ "type": "const_string",
53
+ "value": "</arg_key:opensource>"
54
+ },
55
+ {
56
+ "type": "const_string",
57
+ "value": "\n"
58
+ },
59
+ {
60
+ "type": "const_string",
61
+ "value": "<arg_value:opensource>"
62
+ },
63
+ {
64
+ "type": "any_text",
65
+ "excludes": [
66
+ "<tool_calls:opensource>",
67
+ "</tool_calls:opensource>",
68
+ "<tool_call:opensource>",
69
+ "</tool_call:opensource>",
70
+ "<tool_sep:opensource>",
71
+ "<arg_key:opensource>",
72
+ "</arg_key:opensource>",
73
+ "<arg_value:opensource>",
74
+ "</arg_value:opensource>"
75
+ ],
76
+ "max_tokens": null,
77
+ "max_chars": null
78
+ },
79
+ {
80
+ "type": "const_string",
81
+ "value": "</arg_value:opensource>"
82
+ },
83
+ {
84
+ "type": "const_string",
85
+ "value": "\n"
86
+ }
87
+ ]
88
+ }
89
+ ]
90
+ },
91
+ "end": "</tool_call:opensource>"
92
+ },
93
+ {
94
+ "type": "tag",
95
+ "begin": "<tool_call:opensource>add<tool_sep:opensource>\n",
96
+ "content": {
97
+ "type": "sequence",
98
+ "elements": [
99
+ {
100
+ "type": "sequence",
101
+ "elements": [
102
+ {
103
+ "type": "const_string",
104
+ "value": "<arg_key:opensource>"
105
+ },
106
+ {
107
+ "type": "const_string",
108
+ "value": "y"
109
+ },
110
+ {
111
+ "type": "const_string",
112
+ "value": "</arg_key:opensource>"
113
+ },
114
+ {
115
+ "type": "const_string",
116
+ "value": "\n"
117
+ },
118
+ {
119
+ "type": "const_string",
120
+ "value": "<arg_value:opensource>"
121
+ },
122
+ {
123
+ "type": "any_text",
124
+ "excludes": [
125
+ "<tool_calls:opensource>",
126
+ "</tool_calls:opensource>",
127
+ "<tool_call:opensource>",
128
+ "</tool_call:opensource>",
129
+ "<tool_sep:opensource>",
130
+ "<arg_key:opensource>",
131
+ "</arg_key:opensource>",
132
+ "<arg_value:opensource>",
133
+ "</arg_value:opensource>"
134
+ ],
135
+ "max_tokens": null,
136
+ "max_chars": null
137
+ },
138
+ {
139
+ "type": "const_string",
140
+ "value": "</arg_value:opensource>"
141
+ },
142
+ {
143
+ "type": "const_string",
144
+ "value": "\n"
145
+ }
146
+ ]
147
+ },
148
+ {
149
+ "type": "sequence",
150
+ "elements": [
151
+ {
152
+ "type": "const_string",
153
+ "value": "<arg_key:opensource>"
154
+ },
155
+ {
156
+ "type": "const_string",
157
+ "value": "x"
158
+ },
159
+ {
160
+ "type": "const_string",
161
+ "value": "</arg_key:opensource>"
162
+ },
163
+ {
164
+ "type": "const_string",
165
+ "value": "\n"
166
+ },
167
+ {
168
+ "type": "const_string",
169
+ "value": "<arg_value:opensource>"
170
+ },
171
+ {
172
+ "type": "any_text",
173
+ "excludes": [
174
+ "<tool_calls:opensource>",
175
+ "</tool_calls:opensource>",
176
+ "<tool_call:opensource>",
177
+ "</tool_call:opensource>",
178
+ "<tool_sep:opensource>",
179
+ "<arg_key:opensource>",
180
+ "</arg_key:opensource>",
181
+ "<arg_value:opensource>",
182
+ "</arg_value:opensource>"
183
+ ],
184
+ "max_tokens": null,
185
+ "max_chars": null
186
+ },
187
+ {
188
+ "type": "const_string",
189
+ "value": "</arg_value:opensource>"
190
+ },
191
+ {
192
+ "type": "const_string",
193
+ "value": "\n"
194
+ }
195
+ ]
196
+ },
197
+ {
198
+ "type": "sequence",
199
+ "elements": [
200
+ {
201
+ "type": "const_string",
202
+ "value": "<arg_key:opensource>"
203
+ },
204
+ {
205
+ "type": "const_string",
206
+ "value": "items"
207
+ },
208
+ {
209
+ "type": "const_string",
210
+ "value": "</arg_key:opensource>"
211
+ },
212
+ {
213
+ "type": "const_string",
214
+ "value": "\n"
215
+ },
216
+ {
217
+ "type": "const_string",
218
+ "value": "<arg_value:opensource>"
219
+ },
220
+ {
221
+ "type": "any_text",
222
+ "excludes": [
223
+ "<tool_calls:opensource>",
224
+ "</tool_calls:opensource>",
225
+ "<tool_call:opensource>",
226
+ "</tool_call:opensource>",
227
+ "<tool_sep:opensource>",
228
+ "<arg_key:opensource>",
229
+ "</arg_key:opensource>",
230
+ "<arg_value:opensource>",
231
+ "</arg_value:opensource>"
232
+ ],
233
+ "max_tokens": null,
234
+ "max_chars": null
235
+ },
236
+ {
237
+ "type": "const_string",
238
+ "value": "</arg_value:opensource>"
239
+ },
240
+ {
241
+ "type": "const_string",
242
+ "value": "\n"
243
+ }
244
+ ]
245
+ }
246
+ ]
247
+ },
248
+ "end": "</tool_call:opensource>"
249
+ }
250
+ ],
251
+ "separator": "\n",
252
+ "at_least_one": true,
253
+ "stop_after_first": false
254
+ },
255
+ "end": "\n</tool_calls:opensource>"
256
+ }
257
+ ],
258
+ "at_least_one": false,
259
+ "stop_after_first": false,
260
+ "excludes": []
261
+ }
262
+ ]
263
+ },
264
+ "coverage": "from_token_zero"
265
+ },
266
+ "completion": "Need call the weather and add tools.</think:opensource>I will call the tools.<tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n<tool_call:opensource>add<tool_sep:opensource>\n<arg_key:opensource>y</arg_key:opensource>\n<arg_value:opensource>1.0</arg_value:opensource>\n<arg_key:opensource>x</arg_key:opensource>\n<arg_value:opensource>2</arg_value:opensource>\n<arg_key:opensource>items</arg_key:opensource>\n<arg_value:opensource>[\"left\", \"right\"]</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
267
+ "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025]
268
+ },
269
+ "required": {
270
+ "grammar": {
271
+ "format": {
272
+ "type": "sequence",
273
+ "elements": [
274
+ {
275
+ "type": "tag",
276
+ "begin": "",
277
+ "content": {
278
+ "type": "any_text",
279
+ "excludes": [],
280
+ "max_tokens": null,
281
+ "max_chars": null
282
+ },
283
+ "end": "</think:opensource>"
284
+ },
285
+ {
286
+ "type": "sequence",
287
+ "elements": [
288
+ {
289
+ "type": "const_string",
290
+ "value": "<tool_calls:opensource>\n"
291
+ },
292
+ {
293
+ "type": "tags_with_separator",
294
+ "tags": [
295
+ {
296
+ "type": "tag",
297
+ "begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
298
+ "content": {
299
+ "type": "sequence",
300
+ "elements": [
301
+ {
302
+ "type": "sequence",
303
+ "elements": [
304
+ {
305
+ "type": "const_string",
306
+ "value": "<arg_key:opensource>"
307
+ },
308
+ {
309
+ "type": "const_string",
310
+ "value": "location"
311
+ },
312
+ {
313
+ "type": "const_string",
314
+ "value": "</arg_key:opensource>"
315
+ },
316
+ {
317
+ "type": "const_string",
318
+ "value": "\n"
319
+ },
320
+ {
321
+ "type": "const_string",
322
+ "value": "<arg_value:opensource>"
323
+ },
324
+ {
325
+ "type": "any_text",
326
+ "excludes": [
327
+ "<tool_calls:opensource>",
328
+ "</tool_calls:opensource>",
329
+ "<tool_call:opensource>",
330
+ "</tool_call:opensource>",
331
+ "<tool_sep:opensource>",
332
+ "<arg_key:opensource>",
333
+ "</arg_key:opensource>",
334
+ "<arg_value:opensource>",
335
+ "</arg_value:opensource>"
336
+ ],
337
+ "max_tokens": null,
338
+ "max_chars": null
339
+ },
340
+ {
341
+ "type": "const_string",
342
+ "value": "</arg_value:opensource>"
343
+ },
344
+ {
345
+ "type": "const_string",
346
+ "value": "\n"
347
+ }
348
+ ]
349
+ }
350
+ ]
351
+ },
352
+ "end": "</tool_call:opensource>"
353
+ },
354
+ {
355
+ "type": "tag",
356
+ "begin": "<tool_call:opensource>add<tool_sep:opensource>\n",
357
+ "content": {
358
+ "type": "sequence",
359
+ "elements": [
360
+ {
361
+ "type": "sequence",
362
+ "elements": [
363
+ {
364
+ "type": "const_string",
365
+ "value": "<arg_key:opensource>"
366
+ },
367
+ {
368
+ "type": "const_string",
369
+ "value": "y"
370
+ },
371
+ {
372
+ "type": "const_string",
373
+ "value": "</arg_key:opensource>"
374
+ },
375
+ {
376
+ "type": "const_string",
377
+ "value": "\n"
378
+ },
379
+ {
380
+ "type": "const_string",
381
+ "value": "<arg_value:opensource>"
382
+ },
383
+ {
384
+ "type": "any_text",
385
+ "excludes": [
386
+ "<tool_calls:opensource>",
387
+ "</tool_calls:opensource>",
388
+ "<tool_call:opensource>",
389
+ "</tool_call:opensource>",
390
+ "<tool_sep:opensource>",
391
+ "<arg_key:opensource>",
392
+ "</arg_key:opensource>",
393
+ "<arg_value:opensource>",
394
+ "</arg_value:opensource>"
395
+ ],
396
+ "max_tokens": null,
397
+ "max_chars": null
398
+ },
399
+ {
400
+ "type": "const_string",
401
+ "value": "</arg_value:opensource>"
402
+ },
403
+ {
404
+ "type": "const_string",
405
+ "value": "\n"
406
+ }
407
+ ]
408
+ },
409
+ {
410
+ "type": "sequence",
411
+ "elements": [
412
+ {
413
+ "type": "const_string",
414
+ "value": "<arg_key:opensource>"
415
+ },
416
+ {
417
+ "type": "const_string",
418
+ "value": "x"
419
+ },
420
+ {
421
+ "type": "const_string",
422
+ "value": "</arg_key:opensource>"
423
+ },
424
+ {
425
+ "type": "const_string",
426
+ "value": "\n"
427
+ },
428
+ {
429
+ "type": "const_string",
430
+ "value": "<arg_value:opensource>"
431
+ },
432
+ {
433
+ "type": "any_text",
434
+ "excludes": [
435
+ "<tool_calls:opensource>",
436
+ "</tool_calls:opensource>",
437
+ "<tool_call:opensource>",
438
+ "</tool_call:opensource>",
439
+ "<tool_sep:opensource>",
440
+ "<arg_key:opensource>",
441
+ "</arg_key:opensource>",
442
+ "<arg_value:opensource>",
443
+ "</arg_value:opensource>"
444
+ ],
445
+ "max_tokens": null,
446
+ "max_chars": null
447
+ },
448
+ {
449
+ "type": "const_string",
450
+ "value": "</arg_value:opensource>"
451
+ },
452
+ {
453
+ "type": "const_string",
454
+ "value": "\n"
455
+ }
456
+ ]
457
+ },
458
+ {
459
+ "type": "sequence",
460
+ "elements": [
461
+ {
462
+ "type": "const_string",
463
+ "value": "<arg_key:opensource>"
464
+ },
465
+ {
466
+ "type": "const_string",
467
+ "value": "items"
468
+ },
469
+ {
470
+ "type": "const_string",
471
+ "value": "</arg_key:opensource>"
472
+ },
473
+ {
474
+ "type": "const_string",
475
+ "value": "\n"
476
+ },
477
+ {
478
+ "type": "const_string",
479
+ "value": "<arg_value:opensource>"
480
+ },
481
+ {
482
+ "type": "any_text",
483
+ "excludes": [
484
+ "<tool_calls:opensource>",
485
+ "</tool_calls:opensource>",
486
+ "<tool_call:opensource>",
487
+ "</tool_call:opensource>",
488
+ "<tool_sep:opensource>",
489
+ "<arg_key:opensource>",
490
+ "</arg_key:opensource>",
491
+ "<arg_value:opensource>",
492
+ "</arg_value:opensource>"
493
+ ],
494
+ "max_tokens": null,
495
+ "max_chars": null
496
+ },
497
+ {
498
+ "type": "const_string",
499
+ "value": "</arg_value:opensource>"
500
+ },
501
+ {
502
+ "type": "const_string",
503
+ "value": "\n"
504
+ }
505
+ ]
506
+ }
507
+ ]
508
+ },
509
+ "end": "</tool_call:opensource>"
510
+ }
511
+ ],
512
+ "separator": "\n",
513
+ "at_least_one": true,
514
+ "stop_after_first": false
515
+ },
516
+ {
517
+ "type": "const_string",
518
+ "value": "\n</tool_calls:opensource>"
519
+ }
520
+ ]
521
+ }
522
+ ]
523
+ },
524
+ "coverage": "from_token_zero"
525
+ },
526
+ "completion": "Need call the weather and add tools.</think:opensource><tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n<tool_call:opensource>add<tool_sep:opensource>\n<arg_key:opensource>y</arg_key:opensource>\n<arg_value:opensource>1.0</arg_value:opensource>\n<arg_key:opensource>x</arg_key:opensource>\n<arg_value:opensource>2</arg_value:opensource>\n<arg_key:opensource>items</arg_key:opensource>\n<arg_value:opensource>[\"left\", \"right\"]</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
527
+ "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025]
528
+ },
529
+ "named": {
530
+ "grammar": {
531
+ "format": {
532
+ "type": "sequence",
533
+ "elements": [
534
+ {
535
+ "type": "tag",
536
+ "begin": "",
537
+ "content": {
538
+ "type": "any_text",
539
+ "excludes": [],
540
+ "max_tokens": null,
541
+ "max_chars": null
542
+ },
543
+ "end": "</think:opensource>"
544
+ },
545
+ {
546
+ "type": "sequence",
547
+ "elements": [
548
+ {
549
+ "type": "const_string",
550
+ "value": "<tool_calls:opensource>\n"
551
+ },
552
+ {
553
+ "type": "tag",
554
+ "begin": "<tool_call:opensource>get_weather<tool_sep:opensource>\n",
555
+ "content": {
556
+ "type": "sequence",
557
+ "elements": [
558
+ {
559
+ "type": "sequence",
560
+ "elements": [
561
+ {
562
+ "type": "const_string",
563
+ "value": "<arg_key:opensource>"
564
+ },
565
+ {
566
+ "type": "const_string",
567
+ "value": "location"
568
+ },
569
+ {
570
+ "type": "const_string",
571
+ "value": "</arg_key:opensource>"
572
+ },
573
+ {
574
+ "type": "const_string",
575
+ "value": "\n"
576
+ },
577
+ {
578
+ "type": "const_string",
579
+ "value": "<arg_value:opensource>"
580
+ },
581
+ {
582
+ "type": "any_text",
583
+ "excludes": [
584
+ "<tool_calls:opensource>",
585
+ "</tool_calls:opensource>",
586
+ "<tool_call:opensource>",
587
+ "</tool_call:opensource>",
588
+ "<tool_sep:opensource>",
589
+ "<arg_key:opensource>",
590
+ "</arg_key:opensource>",
591
+ "<arg_value:opensource>",
592
+ "</arg_value:opensource>"
593
+ ],
594
+ "max_tokens": null,
595
+ "max_chars": null
596
+ },
597
+ {
598
+ "type": "const_string",
599
+ "value": "</arg_value:opensource>"
600
+ },
601
+ {
602
+ "type": "const_string",
603
+ "value": "\n"
604
+ }
605
+ ]
606
+ }
607
+ ]
608
+ },
609
+ "end": "</tool_call:opensource>"
610
+ },
611
+ {
612
+ "type": "const_string",
613
+ "value": "\n</tool_calls:opensource>"
614
+ }
615
+ ]
616
+ }
617
+ ]
618
+ },
619
+ "coverage": "from_token_zero"
620
+ },
621
+ "completion": "Need call the weather and add tools.</think:opensource><tool_calls:opensource>\n<tool_call:opensource>get_weather<tool_sep:opensource>\n<arg_key:opensource>location</arg_key:opensource>\n<arg_value:opensource>Shanghai</arg_value:opensource>\n</tool_call:opensource>\n</tool_calls:opensource>",
622
+ "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025]
623
+ }
624
+ }
625
+}
rust/src/chat/tests/grammar_replay/Kimi-K2.5.json
new
+340
@@ -0,0 +1,340 @@
1
+{
2
+ "model_id": "moonshotai/Kimi-K2.5",
3
+ "vocab_size": 163840,
4
+ "all_stop_token_ids": [163585, 163586],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "tag",
13
+ "begin": "",
14
+ "content": {
15
+ "type": "any_text",
16
+ "excludes": [],
17
+ "max_tokens": null,
18
+ "max_chars": null
19
+ },
20
+ "end": "</think>"
21
+ },
22
+ {
23
+ "type": "triggered_tags",
24
+ "triggers": [
25
+ "<|tool_calls_section_begin|>"
26
+ ],
27
+ "tags": [
28
+ {
29
+ "type": "tag",
30
+ "begin": "<|tool_calls_section_begin|>",
31
+ "content": {
32
+ "type": "tags_with_separator",
33
+ "tags": [
34
+ {
35
+ "type": "tag",
36
+ "begin": "<|tool_call_begin|>functions.get_weather:",
37
+ "content": {
38
+ "type": "sequence",
39
+ "elements": [
40
+ {
41
+ "type": "regex",
42
+ "pattern": "\\d+"
43
+ },
44
+ {
45
+ "type": "const_string",
46
+ "value": "<|tool_call_argument_begin|>"
47
+ },
48
+ {
49
+ "type": "json_schema",
50
+ "json_schema": {
51
+ "type": "object",
52
+ "properties": {
53
+ "location": {
54
+ "type": "string"
55
+ }
56
+ },
57
+ "required": [
58
+ "location"
59
+ ]
60
+ },
61
+ "style": "json",
62
+ "any_order": false,
63
+ "max_whitespace_cnt": null,
64
+ "excludes": []
65
+ }
66
+ ]
67
+ },
68
+ "end": "<|tool_call_end|>"
69
+ },
70
+ {
71
+ "type": "tag",
72
+ "begin": "<|tool_call_begin|>functions.add:",
73
+ "content": {
74
+ "type": "sequence",
75
+ "elements": [
76
+ {
77
+ "type": "regex",
78
+ "pattern": "\\d+"
79
+ },
80
+ {
81
+ "type": "const_string",
82
+ "value": "<|tool_call_argument_begin|>"
83
+ },
84
+ {
85
+ "type": "json_schema",
86
+ "json_schema": {
87
+ "type": "object",
88
+ "properties": {
89
+ "y": {
90
+ "type": "number"
91
+ },
92
+ "x": {
93
+ "type": "number"
94
+ },
95
+ "items": {
96
+ "type": "array",
97
+ "items": {
98
+ "type": "string"
99
+ }
100
+ }
101
+ },
102
+ "required": [
103
+ "y",
104
+ "x",
105
+ "items"
106
+ ]
107
+ },
108
+ "style": "json",
109
+ "any_order": false,
110
+ "max_whitespace_cnt": null,
111
+ "excludes": []
112
+ }
113
+ ]
114
+ },
115
+ "end": "<|tool_call_end|>"
116
+ }
117
+ ],
118
+ "separator": "",
119
+ "at_least_one": true,
120
+ "stop_after_first": false
121
+ },
122
+ "end": "<|tool_calls_section_end|>"
123
+ }
124
+ ],
125
+ "at_least_one": false,
126
+ "stop_after_first": false,
127
+ "excludes": [
128
+ "<think>",
129
+ "</think>",
130
+ "<|tool_call_begin|>"
131
+ ]
132
+ }
133
+ ]
134
+ },
135
+ "coverage": "from_token_zero"
136
+ },
137
+ "completion": "Need call the weather and add tools.</think>I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>",
138
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586]
139
+ },
140
+ "required": {
141
+ "grammar": {
142
+ "format": {
143
+ "type": "sequence",
144
+ "elements": [
145
+ {
146
+ "type": "tag",
147
+ "begin": "",
148
+ "content": {
149
+ "type": "any_text",
150
+ "excludes": [],
151
+ "max_tokens": null,
152
+ "max_chars": null
153
+ },
154
+ "end": "</think>"
155
+ },
156
+ {
157
+ "type": "sequence",
158
+ "elements": [
159
+ {
160
+ "type": "const_string",
161
+ "value": "<|tool_calls_section_begin|>"
162
+ },
163
+ {
164
+ "type": "tags_with_separator",
165
+ "tags": [
166
+ {
167
+ "type": "tag",
168
+ "begin": "<|tool_call_begin|>functions.get_weather:",
169
+ "content": {
170
+ "type": "sequence",
171
+ "elements": [
172
+ {
173
+ "type": "regex",
174
+ "pattern": "\\d+"
175
+ },
176
+ {
177
+ "type": "const_string",
178
+ "value": "<|tool_call_argument_begin|>"
179
+ },
180
+ {
181
+ "type": "json_schema",
182
+ "json_schema": {
183
+ "type": "object",
184
+ "properties": {
185
+ "location": {
186
+ "type": "string"
187
+ }
188
+ },
189
+ "required": [
190
+ "location"
191
+ ]
192
+ },
193
+ "style": "json",
194
+ "any_order": false,
195
+ "max_whitespace_cnt": null,
196
+ "excludes": []
197
+ }
198
+ ]
199
+ },
200
+ "end": "<|tool_call_end|>"
201
+ },
202
+ {
203
+ "type": "tag",
204
+ "begin": "<|tool_call_begin|>functions.add:",
205
+ "content": {
206
+ "type": "sequence",
207
+ "elements": [
208
+ {
209
+ "type": "regex",
210
+ "pattern": "\\d+"
211
+ },
212
+ {
213
+ "type": "const_string",
214
+ "value": "<|tool_call_argument_begin|>"
215
+ },
216
+ {
217
+ "type": "json_schema",
218
+ "json_schema": {
219
+ "type": "object",
220
+ "properties": {
221
+ "y": {
222
+ "type": "number"
223
+ },
224
+ "x": {
225
+ "type": "number"
226
+ },
227
+ "items": {
228
+ "type": "array",
229
+ "items": {
230
+ "type": "string"
231
+ }
232
+ }
233
+ },
234
+ "required": [
235
+ "y",
236
+ "x",
237
+ "items"
238
+ ]
239
+ },
240
+ "style": "json",
241
+ "any_order": false,
242
+ "max_whitespace_cnt": null,
243
+ "excludes": []
244
+ }
245
+ ]
246
+ },
247
+ "end": "<|tool_call_end|>"
248
+ }
249
+ ],
250
+ "separator": "",
251
+ "at_least_one": true,
252
+ "stop_after_first": false
253
+ },
254
+ {
255
+ "type": "const_string",
256
+ "value": "<|tool_calls_section_end|>"
257
+ }
258
+ ]
259
+ }
260
+ ]
261
+ },
262
+ "coverage": "from_token_zero"
263
+ },
264
+ "completion": "Need call the weather and add tools.</think><|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>",
265
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586]
266
+ },
267
+ "named": {
268
+ "grammar": {
269
+ "format": {
270
+ "type": "sequence",
271
+ "elements": [
272
+ {
273
+ "type": "tag",
274
+ "begin": "",
275
+ "content": {
276
+ "type": "any_text",
277
+ "excludes": [],
278
+ "max_tokens": null,
279
+ "max_chars": null
280
+ },
281
+ "end": "</think>"
282
+ },
283
+ {
284
+ "type": "sequence",
285
+ "elements": [
286
+ {
287
+ "type": "const_string",
288
+ "value": "<|tool_calls_section_begin|>"
289
+ },
290
+ {
291
+ "type": "tag",
292
+ "begin": "<|tool_call_begin|>functions.get_weather:",
293
+ "content": {
294
+ "type": "sequence",
295
+ "elements": [
296
+ {
297
+ "type": "regex",
298
+ "pattern": "\\d+"
299
+ },
300
+ {
301
+ "type": "const_string",
302
+ "value": "<|tool_call_argument_begin|>"
303
+ },
304
+ {
305
+ "type": "json_schema",
306
+ "json_schema": {
307
+ "type": "object",
308
+ "properties": {
309
+ "location": {
310
+ "type": "string"
311
+ }
312
+ },
313
+ "required": [
314
+ "location"
315
+ ]
316
+ },
317
+ "style": "json",
318
+ "any_order": false,
319
+ "max_whitespace_cnt": null,
320
+ "excludes": []
321
+ }
322
+ ]
323
+ },
324
+ "end": "<|tool_call_end|>"
325
+ },
326
+ {
327
+ "type": "const_string",
328
+ "value": "<|tool_calls_section_end|>"
329
+ }
330
+ ]
331
+ }
332
+ ]
333
+ },
334
+ "coverage": "from_token_zero"
335
+ },
336
+ "completion": "Need call the weather and add tools.</think><|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>",
337
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586]
338
+ }
339
+ }
340
+}
rust/src/chat/tests/grammar_replay/Kimi-K3.json
new
+587
@@ -0,0 +1,587 @@
1
+{
2
+ "model_id": "moonshotai/Kimi-K3",
3
+ "vocab_size": 163840,
4
+ "all_stop_token_ids": [163585, 163586],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "optional",
13
+ "content": {
14
+ "type": "const_string",
15
+ "value": "<|open|>response<|sep|>"
16
+ }
17
+ },
18
+ {
19
+ "type": "tag",
20
+ "begin": "",
21
+ "content": {
22
+ "type": "any_text",
23
+ "excludes": [
24
+ "<|open|>",
25
+ "<|close|>",
26
+ "<|end_of_msg|>"
27
+ ],
28
+ "max_tokens": null,
29
+ "max_chars": null
30
+ },
31
+ "end": "<|close|>response<|sep|>"
32
+ },
33
+ {
34
+ "type": "optional",
35
+ "content": {
36
+ "type": "tag",
37
+ "begin": "<|open|>tools<|sep|>",
38
+ "content": {
39
+ "type": "tags_with_separator",
40
+ "tags": [
41
+ {
42
+ "type": "tag",
43
+ "begin": "<|open|>call tool=\"get_weather\" index=\"",
44
+ "content": {
45
+ "type": "sequence",
46
+ "elements": [
47
+ {
48
+ "type": "regex",
49
+ "pattern": "[1-9][0-9]*"
50
+ },
51
+ {
52
+ "type": "const_string",
53
+ "value": "\"<|sep|>"
54
+ },
55
+ {
56
+ "type": "or",
57
+ "elements": [
58
+ {
59
+ "type": "plus",
60
+ "content": {
61
+ "type": "tag",
62
+ "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
63
+ "content": {
64
+ "type": "any_text",
65
+ "excludes": [
66
+ "<|close|>argument<|sep|>",
67
+ "<|close|>call<|sep|>"
68
+ ],
69
+ "max_tokens": null,
70
+ "max_chars": null
71
+ },
72
+ "end": "<|close|>argument<|sep|>"
73
+ }
74
+ },
75
+ {
76
+ "type": "tag",
77
+ "begin": "<|open|>json type=\"object\"<|sep|>",
78
+ "content": {
79
+ "type": "json_schema",
80
+ "json_schema": {
81
+ "type": "object",
82
+ "properties": {
83
+ "location": {
84
+ "type": "string"
85
+ }
86
+ },
87
+ "required": [
88
+ "location"
89
+ ]
90
+ },
91
+ "style": "json",
92
+ "any_order": false,
93
+ "max_whitespace_cnt": null,
94
+ "excludes": []
95
+ },
96
+ "end": "<|close|>json<|sep|>"
97
+ }
98
+ ]
99
+ }
100
+ ]
101
+ },
102
+ "end": "<|close|>call<|sep|>"
103
+ },
104
+ {
105
+ "type": "tag",
106
+ "begin": "<|open|>call tool=\"add\" index=\"",
107
+ "content": {
108
+ "type": "sequence",
109
+ "elements": [
110
+ {
111
+ "type": "regex",
112
+ "pattern": "[1-9][0-9]*"
113
+ },
114
+ {
115
+ "type": "const_string",
116
+ "value": "\"<|sep|>"
117
+ },
118
+ {
119
+ "type": "or",
120
+ "elements": [
121
+ {
122
+ "type": "plus",
123
+ "content": {
124
+ "type": "or",
125
+ "elements": [
126
+ {
127
+ "type": "tag",
128
+ "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>",
129
+ "content": {
130
+ "type": "json_schema",
131
+ "json_schema": {
132
+ "type": "number"
133
+ },
134
+ "style": "json",
135
+ "any_order": false,
136
+ "max_whitespace_cnt": null,
137
+ "excludes": []
138
+ },
139
+ "end": "<|close|>argument<|sep|>"
140
+ },
141
+ {
142
+ "type": "tag",
143
+ "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>",
144
+ "content": {
145
+ "type": "json_schema",
146
+ "json_schema": {
147
+ "type": "number"
148
+ },
149
+ "style": "json",
150
+ "any_order": false,
151
+ "max_whitespace_cnt": null,
152
+ "excludes": []
153
+ },
154
+ "end": "<|close|>argument<|sep|>"
155
+ },
156
+ {
157
+ "type": "tag",
158
+ "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>",
159
+ "content": {
160
+ "type": "json_schema",
161
+ "json_schema": {
162
+ "type": "array",
163
+ "items": {
164
+ "type": "string"
165
+ }
166
+ },
167
+ "style": "json",
168
+ "any_order": false,
169
+ "max_whitespace_cnt": null,
170
+ "excludes": []
171
+ },
172
+ "end": "<|close|>argument<|sep|>"
173
+ }
174
+ ]
175
+ }
176
+ },
177
+ {
178
+ "type": "tag",
179
+ "begin": "<|open|>json type=\"object\"<|sep|>",
180
+ "content": {
181
+ "type": "json_schema",
182
+ "json_schema": {
183
+ "type": "object",
184
+ "properties": {
185
+ "y": {
186
+ "type": "number"
187
+ },
188
+ "x": {
189
+ "type": "number"
190
+ },
191
+ "items": {
192
+ "type": "array",
193
+ "items": {
194
+ "type": "string"
195
+ }
196
+ }
197
+ },
198
+ "required": [
199
+ "y",
200
+ "x",
201
+ "items"
202
+ ]
203
+ },
204
+ "style": "json",
205
+ "any_order": false,
206
+ "max_whitespace_cnt": null,
207
+ "excludes": []
208
+ },
209
+ "end": "<|close|>json<|sep|>"
210
+ }
211
+ ]
212
+ }
213
+ ]
214
+ },
215
+ "end": "<|close|>call<|sep|>"
216
+ }
217
+ ],
218
+ "separator": "",
219
+ "at_least_one": true,
220
+ "stop_after_first": false
221
+ },
222
+ "end": "<|close|>tools<|sep|>"
223
+ }
224
+ },
225
+ {
226
+ "type": "optional",
227
+ "content": {
228
+ "type": "const_string",
229
+ "value": "<|close|>message<|sep|>"
230
+ }
231
+ }
232
+ ]
233
+ },
234
+ "coverage": "final_output_only"
235
+ },
236
+ "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
237
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
238
+ },
239
+ "required": {
240
+ "grammar": {
241
+ "format": {
242
+ "type": "sequence",
243
+ "elements": [
244
+ {
245
+ "type": "optional",
246
+ "content": {
247
+ "type": "const_string",
248
+ "value": "<|open|>response<|sep|>"
249
+ }
250
+ },
251
+ {
252
+ "type": "tag",
253
+ "begin": "",
254
+ "content": {
255
+ "type": "any_text",
256
+ "excludes": [
257
+ "<|open|>",
258
+ "<|close|>",
259
+ "<|end_of_msg|>"
260
+ ],
261
+ "max_tokens": null,
262
+ "max_chars": null
263
+ },
264
+ "end": "<|close|>response<|sep|>"
265
+ },
266
+ {
267
+ "type": "tag",
268
+ "begin": "<|open|>tools<|sep|>",
269
+ "content": {
270
+ "type": "tags_with_separator",
271
+ "tags": [
272
+ {
273
+ "type": "tag",
274
+ "begin": "<|open|>call tool=\"get_weather\" index=\"",
275
+ "content": {
276
+ "type": "sequence",
277
+ "elements": [
278
+ {
279
+ "type": "regex",
280
+ "pattern": "[1-9][0-9]*"
281
+ },
282
+ {
283
+ "type": "const_string",
284
+ "value": "\"<|sep|>"
285
+ },
286
+ {
287
+ "type": "or",
288
+ "elements": [
289
+ {
290
+ "type": "plus",
291
+ "content": {
292
+ "type": "tag",
293
+ "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
294
+ "content": {
295
+ "type": "any_text",
296
+ "excludes": [
297
+ "<|close|>argument<|sep|>",
298
+ "<|close|>call<|sep|>"
299
+ ],
300
+ "max_tokens": null,
301
+ "max_chars": null
302
+ },
303
+ "end": "<|close|>argument<|sep|>"
304
+ }
305
+ },
306
+ {
307
+ "type": "tag",
308
+ "begin": "<|open|>json type=\"object\"<|sep|>",
309
+ "content": {
310
+ "type": "json_schema",
311
+ "json_schema": {
312
+ "type": "object",
313
+ "properties": {
314
+ "location": {
315
+ "type": "string"
316
+ }
317
+ },
318
+ "required": [
319
+ "location"
320
+ ]
321
+ },
322
+ "style": "json",
323
+ "any_order": false,
324
+ "max_whitespace_cnt": null,
325
+ "excludes": []
326
+ },
327
+ "end": "<|close|>json<|sep|>"
328
+ }
329
+ ]
330
+ }
331
+ ]
332
+ },
333
+ "end": "<|close|>call<|sep|>"
334
+ },
335
+ {
336
+ "type": "tag",
337
+ "begin": "<|open|>call tool=\"add\" index=\"",
338
+ "content": {
339
+ "type": "sequence",
340
+ "elements": [
341
+ {
342
+ "type": "regex",
343
+ "pattern": "[1-9][0-9]*"
344
+ },
345
+ {
346
+ "type": "const_string",
347
+ "value": "\"<|sep|>"
348
+ },
349
+ {
350
+ "type": "or",
351
+ "elements": [
352
+ {
353
+ "type": "plus",
354
+ "content": {
355
+ "type": "or",
356
+ "elements": [
357
+ {
358
+ "type": "tag",
359
+ "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>",
360
+ "content": {
361
+ "type": "json_schema",
362
+ "json_schema": {
363
+ "type": "number"
364
+ },
365
+ "style": "json",
366
+ "any_order": false,
367
+ "max_whitespace_cnt": null,
368
+ "excludes": []
369
+ },
370
+ "end": "<|close|>argument<|sep|>"
371
+ },
372
+ {
373
+ "type": "tag",
374
+ "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>",
375
+ "content": {
376
+ "type": "json_schema",
377
+ "json_schema": {
378
+ "type": "number"
379
+ },
380
+ "style": "json",
381
+ "any_order": false,
382
+ "max_whitespace_cnt": null,
383
+ "excludes": []
384
+ },
385
+ "end": "<|close|>argument<|sep|>"
386
+ },
387
+ {
388
+ "type": "tag",
389
+ "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>",
390
+ "content": {
391
+ "type": "json_schema",
392
+ "json_schema": {
393
+ "type": "array",
394
+ "items": {
395
+ "type": "string"
396
+ }
397
+ },
398
+ "style": "json",
399
+ "any_order": false,
400
+ "max_whitespace_cnt": null,
401
+ "excludes": []
402
+ },
403
+ "end": "<|close|>argument<|sep|>"
404
+ }
405
+ ]
406
+ }
407
+ },
408
+ {
409
+ "type": "tag",
410
+ "begin": "<|open|>json type=\"object\"<|sep|>",
411
+ "content": {
412
+ "type": "json_schema",
413
+ "json_schema": {
414
+ "type": "object",
415
+ "properties": {
416
+ "y": {
417
+ "type": "number"
418
+ },
419
+ "x": {
420
+ "type": "number"
421
+ },
422
+ "items": {
423
+ "type": "array",
424
+ "items": {
425
+ "type": "string"
426
+ }
427
+ }
428
+ },
429
+ "required": [
430
+ "y",
431
+ "x",
432
+ "items"
433
+ ]
434
+ },
435
+ "style": "json",
436
+ "any_order": false,
437
+ "max_whitespace_cnt": null,
438
+ "excludes": []
439
+ },
440
+ "end": "<|close|>json<|sep|>"
441
+ }
442
+ ]
443
+ }
444
+ ]
445
+ },
446
+ "end": "<|close|>call<|sep|>"
447
+ }
448
+ ],
449
+ "separator": "",
450
+ "at_least_one": true,
451
+ "stop_after_first": false
452
+ },
453
+ "end": "<|close|>tools<|sep|>"
454
+ },
455
+ {
456
+ "type": "optional",
457
+ "content": {
458
+ "type": "const_string",
459
+ "value": "<|close|>message<|sep|>"
460
+ }
461
+ }
462
+ ]
463
+ },
464
+ "coverage": "final_output_only"
465
+ },
466
+ "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
467
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
468
+ },
469
+ "named": {
470
+ "grammar": {
471
+ "format": {
472
+ "type": "sequence",
473
+ "elements": [
474
+ {
475
+ "type": "optional",
476
+ "content": {
477
+ "type": "const_string",
478
+ "value": "<|open|>response<|sep|>"
479
+ }
480
+ },
481
+ {
482
+ "type": "tag",
483
+ "begin": "",
484
+ "content": {
485
+ "type": "any_text",
486
+ "excludes": [
487
+ "<|open|>",
488
+ "<|close|>",
489
+ "<|end_of_msg|>"
490
+ ],
491
+ "max_tokens": null,
492
+ "max_chars": null
493
+ },
494
+ "end": "<|close|>response<|sep|>"
495
+ },
496
+ {
497
+ "type": "tag",
498
+ "begin": "<|open|>tools<|sep|>",
499
+ "content": {
500
+ "type": "tags_with_separator",
501
+ "tags": [
502
+ {
503
+ "type": "tag",
504
+ "begin": "<|open|>call tool=\"get_weather\" index=\"",
505
+ "content": {
506
+ "type": "sequence",
507
+ "elements": [
508
+ {
509
+ "type": "regex",
510
+ "pattern": "[1-9][0-9]*"
511
+ },
512
+ {
513
+ "type": "const_string",
514
+ "value": "\"<|sep|>"
515
+ },
516
+ {
517
+ "type": "or",
518
+ "elements": [
519
+ {
520
+ "type": "plus",
521
+ "content": {
522
+ "type": "tag",
523
+ "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>",
524
+ "content": {
525
+ "type": "any_text",
526
+ "excludes": [
527
+ "<|close|>argument<|sep|>",
528
+ "<|close|>call<|sep|>"
529
+ ],
530
+ "max_tokens": null,
531
+ "max_chars": null
532
+ },
533
+ "end": "<|close|>argument<|sep|>"
534
+ }
535
+ },
536
+ {
537
+ "type": "tag",
538
+ "begin": "<|open|>json type=\"object\"<|sep|>",
539
+ "content": {
540
+ "type": "json_schema",
541
+ "json_schema": {
542
+ "type": "object",
543
+ "properties": {
544
+ "location": {
545
+ "type": "string"
546
+ }
547
+ },
548
+ "required": [
549
+ "location"
550
+ ]
551
+ },
552
+ "style": "json",
553
+ "any_order": false,
554
+ "max_whitespace_cnt": null,
555
+ "excludes": []
556
+ },
557
+ "end": "<|close|>json<|sep|>"
558
+ }
559
+ ]
560
+ }
561
+ ]
562
+ },
563
+ "end": "<|close|>call<|sep|>"
564
+ }
565
+ ],
566
+ "separator": "",
567
+ "at_least_one": true,
568
+ "stop_after_first": false
569
+ },
570
+ "end": "<|close|>tools<|sep|>"
571
+ },
572
+ {
573
+ "type": "optional",
574
+ "content": {
575
+ "type": "const_string",
576
+ "value": "<|close|>message<|sep|>"
577
+ }
578
+ }
579
+ ]
580
+ },
581
+ "coverage": "final_output_only"
582
+ },
583
+ "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>",
584
+ "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586]
585
+ }
586
+ }
587
+}
rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json
new
+254
@@ -0,0 +1,254 @@
1
+{
2
+ "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL",
3
+ "vocab_size": 152576,
4
+ "all_stop_token_ids": [151643, 151645, 151672],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "optional",
13
+ "content": {
14
+ "type": "tag",
15
+ "begin": "<think>",
16
+ "content": {
17
+ "type": "any_text",
18
+ "excludes": [],
19
+ "max_tokens": null,
20
+ "max_chars": null
21
+ },
22
+ "end": "</think>"
23
+ }
24
+ },
25
+ {
26
+ "type": "triggered_tags",
27
+ "triggers": [
28
+ "<tool_call>"
29
+ ],
30
+ "tags": [
31
+ {
32
+ "type": "tag",
33
+ "begin": "<tool_call><function=get_weather>",
34
+ "content": {
35
+ "type": "json_schema",
36
+ "json_schema": {
37
+ "type": "object",
38
+ "properties": {
39
+ "location": {
40
+ "type": "string"
41
+ }
42
+ },
43
+ "required": [
44
+ "location"
45
+ ]
46
+ },
47
+ "style": "qwen_xml",
48
+ "any_order": false,
49
+ "max_whitespace_cnt": null,
50
+ "excludes": []
51
+ },
52
+ "end": "</function></tool_call>"
53
+ },
54
+ {
55
+ "type": "tag",
56
+ "begin": "<tool_call><function=add>",
57
+ "content": {
58
+ "type": "json_schema",
59
+ "json_schema": {
60
+ "type": "object",
61
+ "properties": {
62
+ "y": {
63
+ "type": "number"
64
+ },
65
+ "x": {
66
+ "type": "number"
67
+ },
68
+ "items": {
69
+ "type": "array",
70
+ "items": {
71
+ "type": "string"
72
+ }
73
+ }
74
+ },
75
+ "required": [
76
+ "y",
77
+ "x",
78
+ "items"
79
+ ]
80
+ },
81
+ "style": "qwen_xml",
82
+ "any_order": false,
83
+ "max_whitespace_cnt": null,
84
+ "excludes": []
85
+ },
86
+ "end": "</function></tool_call>"
87
+ }
88
+ ],
89
+ "at_least_one": false,
90
+ "stop_after_first": false,
91
+ "excludes": [
92
+ "<think>",
93
+ "</think>",
94
+ "</tool_call>",
95
+ "<function="
96
+ ]
97
+ }
98
+ ]
99
+ },
100
+ "coverage": "from_token_zero"
101
+ },
102
+ "completion": "<think>Need call the weather and add tools.</think>I will call the tools.<tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call><tool_call><function=add><parameter=y>1.0</parameter><parameter=x>2</parameter><parameter=items>[\"left\", \"right\"]</parameter></function></tool_call>",
103
+ "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645]
104
+ },
105
+ "required": {
106
+ "grammar": {
107
+ "format": {
108
+ "type": "sequence",
109
+ "elements": [
110
+ {
111
+ "type": "optional",
112
+ "content": {
113
+ "type": "tag",
114
+ "begin": "<think>",
115
+ "content": {
116
+ "type": "any_text",
117
+ "excludes": [],
118
+ "max_tokens": null,
119
+ "max_chars": null
120
+ },
121
+ "end": "</think>"
122
+ }
123
+ },
124
+ {
125
+ "type": "triggered_tags",
126
+ "triggers": [
127
+ "<tool_call>"
128
+ ],
129
+ "tags": [
130
+ {
131
+ "type": "tag",
132
+ "begin": "<tool_call><function=get_weather>",
133
+ "content": {
134
+ "type": "json_schema",
135
+ "json_schema": {
136
+ "type": "object",
137
+ "properties": {
138
+ "location": {
139
+ "type": "string"
140
+ }
141
+ },
142
+ "required": [
143
+ "location"
144
+ ]
145
+ },
146
+ "style": "qwen_xml",
147
+ "any_order": false,
148
+ "max_whitespace_cnt": null,
149
+ "excludes": []
150
+ },
151
+ "end": "</function></tool_call>"
152
+ },
153
+ {
154
+ "type": "tag",
155
+ "begin": "<tool_call><function=add>",
156
+ "content": {
157
+ "type": "json_schema",
158
+ "json_schema": {
159
+ "type": "object",
160
+ "properties": {
161
+ "y": {
162
+ "type": "number"
163
+ },
164
+ "x": {
165
+ "type": "number"
166
+ },
167
+ "items": {
168
+ "type": "array",
169
+ "items": {
170
+ "type": "string"
171
+ }
172
+ }
173
+ },
174
+ "required": [
175
+ "y",
176
+ "x",
177
+ "items"
178
+ ]
179
+ },
180
+ "style": "qwen_xml",
181
+ "any_order": false,
182
+ "max_whitespace_cnt": null,
183
+ "excludes": []
184
+ },
185
+ "end": "</function></tool_call>"
186
+ }
187
+ ],
188
+ "at_least_one": true,
189
+ "stop_after_first": false,
190
+ "excludes": [
191
+ "<think>",
192
+ "</think>",
193
+ "</tool_call>",
194
+ "<function="
195
+ ]
196
+ }
197
+ ]
198
+ },
199
+ "coverage": "from_token_zero"
200
+ },
201
+ "completion": "<think>Need call the weather and add tools.</think><tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call><tool_call><function=add><parameter=y>1.0</parameter><parameter=x>2</parameter><parameter=items>[\"left\", \"right\"]</parameter></function></tool_call>",
202
+ "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645]
203
+ },
204
+ "named": {
205
+ "grammar": {
206
+ "format": {
207
+ "type": "sequence",
208
+ "elements": [
209
+ {
210
+ "type": "optional",
211
+ "content": {
212
+ "type": "tag",
213
+ "begin": "<think>",
214
+ "content": {
215
+ "type": "any_text",
216
+ "excludes": [],
217
+ "max_tokens": null,
218
+ "max_chars": null
219
+ },
220
+ "end": "</think>"
221
+ }
222
+ },
223
+ {
224
+ "type": "tag",
225
+ "begin": "<tool_call><function=get_weather>",
226
+ "content": {
227
+ "type": "json_schema",
228
+ "json_schema": {
229
+ "type": "object",
230
+ "properties": {
231
+ "location": {
232
+ "type": "string"
233
+ }
234
+ },
235
+ "required": [
236
+ "location"
237
+ ]
238
+ },
239
+ "style": "qwen_xml",
240
+ "any_order": false,
241
+ "max_whitespace_cnt": null,
242
+ "excludes": []
243
+ },
244
+ "end": "</function></tool_call>"
245
+ }
246
+ ]
247
+ },
248
+ "coverage": "from_token_zero"
249
+ },
250
+ "completion": "<think>Need call the weather and add tools.</think><tool_call><function=get_weather><parameter=location>Shanghai</parameter></function></tool_call>",
251
+ "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645]
252
+ }
253
+ }
254
+}
rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json
new
+301
@@ -0,0 +1,301 @@
1
+{
2
+ "model_id": "MiniMaxAI/MiniMax-M2.5",
3
+ "vocab_size": 200064,
4
+ "all_stop_token_ids": [200020],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "sequence",
13
+ "elements": [
14
+ {
15
+ "type": "tag",
16
+ "begin": "",
17
+ "content": {
18
+ "type": "any_text",
19
+ "excludes": [],
20
+ "max_tokens": null,
21
+ "max_chars": null
22
+ },
23
+ "end": "</think>"
24
+ },
25
+ {
26
+ "type": "const_string",
27
+ "value": "\n\n"
28
+ }
29
+ ]
30
+ },
31
+ {
32
+ "type": "triggered_tags",
33
+ "triggers": [
34
+ "<minimax:tool_call>"
35
+ ],
36
+ "tags": [
37
+ {
38
+ "type": "tag",
39
+ "begin": "<minimax:tool_call>\n",
40
+ "content": {
41
+ "type": "tags_with_separator",
42
+ "tags": [
43
+ {
44
+ "type": "tag",
45
+ "begin": "<invoke name=\"get_weather\">\n",
46
+ "content": {
47
+ "type": "json_schema",
48
+ "json_schema": {
49
+ "type": "object",
50
+ "properties": {
51
+ "location": {
52
+ "type": "string"
53
+ }
54
+ },
55
+ "required": [
56
+ "location"
57
+ ]
58
+ },
59
+ "style": "minimax_xml",
60
+ "any_order": false,
61
+ "max_whitespace_cnt": null,
62
+ "excludes": []
63
+ },
64
+ "end": "</invoke>\n"
65
+ },
66
+ {
67
+ "type": "tag",
68
+ "begin": "<invoke name=\"add\">\n",
69
+ "content": {
70
+ "type": "json_schema",
71
+ "json_schema": {
72
+ "type": "object",
73
+ "properties": {
74
+ "y": {
75
+ "type": "number"
76
+ },
77
+ "x": {
78
+ "type": "number"
79
+ },
80
+ "items": {
81
+ "type": "array",
82
+ "items": {
83
+ "type": "string"
84
+ }
85
+ }
86
+ },
87
+ "required": [
88
+ "y",
89
+ "x",
90
+ "items"
91
+ ]
92
+ },
93
+ "style": "minimax_xml",
94
+ "any_order": false,
95
+ "max_whitespace_cnt": null,
96
+ "excludes": []
97
+ },
98
+ "end": "</invoke>\n"
99
+ }
100
+ ],
101
+ "separator": "",
102
+ "at_least_one": true,
103
+ "stop_after_first": false
104
+ },
105
+ "end": "</minimax:tool_call>"
106
+ }
107
+ ],
108
+ "at_least_one": false,
109
+ "stop_after_first": false,
110
+ "excludes": [
111
+ "<think>",
112
+ "</think>"
113
+ ]
114
+ }
115
+ ]
116
+ },
117
+ "coverage": "from_token_zero"
118
+ },
119
+ "completion": "Need call the weather and add tools.\n</think>\n\nI will call the tools.\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n<invoke name=\"add\">\n<parameter name=\"y\">1.0</parameter>\n<parameter name=\"x\">2</parameter>\n<parameter name=\"items\">[\"left\", \"right\"]</parameter>\n</invoke>\n</minimax:tool_call>",
120
+ "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
121
+ },
122
+ "required": {
123
+ "grammar": {
124
+ "format": {
125
+ "type": "sequence",
126
+ "elements": [
127
+ {
128
+ "type": "sequence",
129
+ "elements": [
130
+ {
131
+ "type": "tag",
132
+ "begin": "",
133
+ "content": {
134
+ "type": "any_text",
135
+ "excludes": [],
136
+ "max_tokens": null,
137
+ "max_chars": null
138
+ },
139
+ "end": "</think>"
140
+ },
141
+ {
142
+ "type": "const_string",
143
+ "value": "\n\n"
144
+ }
145
+ ]
146
+ },
147
+ {
148
+ "type": "sequence",
149
+ "elements": [
150
+ {
151
+ "type": "const_string",
152
+ "value": "\n<minimax:tool_call>\n"
153
+ },
154
+ {
155
+ "type": "tags_with_separator",
156
+ "tags": [
157
+ {
158
+ "type": "tag",
159
+ "begin": "<invoke name=\"get_weather\">\n",
160
+ "content": {
161
+ "type": "json_schema",
162
+ "json_schema": {
163
+ "type": "object",
164
+ "properties": {
165
+ "location": {
166
+ "type": "string"
167
+ }
168
+ },
169
+ "required": [
170
+ "location"
171
+ ]
172
+ },
173
+ "style": "minimax_xml",
174
+ "any_order": false,
175
+ "max_whitespace_cnt": null,
176
+ "excludes": []
177
+ },
178
+ "end": "</invoke>\n"
179
+ },
180
+ {
181
+ "type": "tag",
182
+ "begin": "<invoke name=\"add\">\n",
183
+ "content": {
184
+ "type": "json_schema",
185
+ "json_schema": {
186
+ "type": "object",
187
+ "properties": {
188
+ "y": {
189
+ "type": "number"
190
+ },
191
+ "x": {
192
+ "type": "number"
193
+ },
194
+ "items": {
195
+ "type": "array",
196
+ "items": {
197
+ "type": "string"
198
+ }
199
+ }
200
+ },
201
+ "required": [
202
+ "y",
203
+ "x",
204
+ "items"
205
+ ]
206
+ },
207
+ "style": "minimax_xml",
208
+ "any_order": false,
209
+ "max_whitespace_cnt": null,
210
+ "excludes": []
211
+ },
212
+ "end": "</invoke>\n"
213
+ }
214
+ ],
215
+ "separator": "",
216
+ "at_least_one": true,
217
+ "stop_after_first": false
218
+ },
219
+ {
220
+ "type": "const_string",
221
+ "value": "</minimax:tool_call>"
222
+ }
223
+ ]
224
+ }
225
+ ]
226
+ },
227
+ "coverage": "from_token_zero"
228
+ },
229
+ "completion": "Need call the weather and add tools.\n</think>\n\n\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n<invoke name=\"add\">\n<parameter name=\"y\">1.0</parameter>\n<parameter name=\"x\">2</parameter>\n<parameter name=\"items\">[\"left\", \"right\"]</parameter>\n</invoke>\n</minimax:tool_call>",
230
+ "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
231
+ },
232
+ "named": {
233
+ "grammar": {
234
+ "format": {
235
+ "type": "sequence",
236
+ "elements": [
237
+ {
238
+ "type": "sequence",
239
+ "elements": [
240
+ {
241
+ "type": "tag",
242
+ "begin": "",
243
+ "content": {
244
+ "type": "any_text",
245
+ "excludes": [],
246
+ "max_tokens": null,
247
+ "max_chars": null
248
+ },
249
+ "end": "</think>"
250
+ },
251
+ {
252
+ "type": "const_string",
253
+ "value": "\n\n"
254
+ }
255
+ ]
256
+ },
257
+ {
258
+ "type": "sequence",
259
+ "elements": [
260
+ {
261
+ "type": "const_string",
262
+ "value": "\n<minimax:tool_call>\n"
263
+ },
264
+ {
265
+ "type": "tag",
266
+ "begin": "<invoke name=\"get_weather\">\n",
267
+ "content": {
268
+ "type": "json_schema",
269
+ "json_schema": {
270
+ "type": "object",
271
+ "properties": {
272
+ "location": {
273
+ "type": "string"
274
+ }
275
+ },
276
+ "required": [
277
+ "location"
278
+ ]
279
+ },
280
+ "style": "minimax_xml",
281
+ "any_order": false,
282
+ "max_whitespace_cnt": null,
283
+ "excludes": []
284
+ },
285
+ "end": "</invoke>\n"
286
+ },
287
+ {
288
+ "type": "const_string",
289
+ "value": "</minimax:tool_call>"
290
+ }
291
+ ]
292
+ }
293
+ ]
294
+ },
295
+ "coverage": "from_token_zero"
296
+ },
297
+ "completion": "Need call the weather and add tools.\n</think>\n\n\n<minimax:tool_call>\n<invoke name=\"get_weather\">\n<parameter name=\"location\">Shanghai</parameter>\n</invoke>\n</minimax:tool_call>",
298
+ "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020]
299
+ }
300
+ }
301
+}
rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json
new
+277
@@ -0,0 +1,277 @@
1
+{
2
+ "model_id": "Qwen/Qwen3-0.6B",
3
+ "vocab_size": 151936,
4
+ "all_stop_token_ids": [151643, 151645],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "optional",
13
+ "content": {
14
+ "type": "sequence",
15
+ "elements": [
16
+ {
17
+ "type": "tag",
18
+ "begin": "<think>",
19
+ "content": {
20
+ "type": "any_text",
21
+ "excludes": [],
22
+ "max_tokens": null,
23
+ "max_chars": null
24
+ },
25
+ "end": "</think>"
26
+ },
27
+ {
28
+ "type": "const_string",
29
+ "value": "\n\n"
30
+ }
31
+ ]
32
+ }
33
+ },
34
+ {
35
+ "type": "triggered_tags",
36
+ "triggers": [
37
+ "<tool_call>"
38
+ ],
39
+ "tags": [
40
+ {
41
+ "type": "tag",
42
+ "begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
43
+ "content": {
44
+ "type": "json_schema",
45
+ "json_schema": {
46
+ "type": "object",
47
+ "properties": {
48
+ "location": {
49
+ "type": "string"
50
+ }
51
+ },
52
+ "required": [
53
+ "location"
54
+ ]
55
+ },
56
+ "style": "json",
57
+ "any_order": false,
58
+ "max_whitespace_cnt": null,
59
+ "excludes": []
60
+ },
61
+ "end": "}\n</tool_call>"
62
+ },
63
+ {
64
+ "type": "tag",
65
+ "begin": "<tool_call>\n{\"name\": \"add\", \"arguments\": ",
66
+ "content": {
67
+ "type": "json_schema",
68
+ "json_schema": {
69
+ "type": "object",
70
+ "properties": {
71
+ "y": {
72
+ "type": "number"
73
+ },
74
+ "x": {
75
+ "type": "number"
76
+ },
77
+ "items": {
78
+ "type": "array",
79
+ "items": {
80
+ "type": "string"
81
+ }
82
+ }
83
+ },
84
+ "required": [
85
+ "y",
86
+ "x",
87
+ "items"
88
+ ]
89
+ },
90
+ "style": "json",
91
+ "any_order": false,
92
+ "max_whitespace_cnt": null,
93
+ "excludes": []
94
+ },
95
+ "end": "}\n</tool_call>"
96
+ }
97
+ ],
98
+ "at_least_one": false,
99
+ "stop_after_first": false,
100
+ "excludes": [
101
+ "<think>",
102
+ "</think>"
103
+ ]
104
+ }
105
+ ]
106
+ },
107
+ "coverage": "from_token_zero"
108
+ },
109
+ "completion": "<think>\nNeed call the weather and add tools.\n</think>\n\nI will call the tools.\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>\n<tool_call>\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n</tool_call>",
110
+ "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645]
111
+ },
112
+ "required": {
113
+ "grammar": {
114
+ "format": {
115
+ "type": "sequence",
116
+ "elements": [
117
+ {
118
+ "type": "optional",
119
+ "content": {
120
+ "type": "sequence",
121
+ "elements": [
122
+ {
123
+ "type": "tag",
124
+ "begin": "<think>",
125
+ "content": {
126
+ "type": "any_text",
127
+ "excludes": [],
128
+ "max_tokens": null,
129
+ "max_chars": null
130
+ },
131
+ "end": "</think>"
132
+ },
133
+ {
134
+ "type": "const_string",
135
+ "value": "\n\n"
136
+ }
137
+ ]
138
+ }
139
+ },
140
+ {
141
+ "type": "triggered_tags",
142
+ "triggers": [
143
+ "<tool_call>"
144
+ ],
145
+ "tags": [
146
+ {
147
+ "type": "tag",
148
+ "begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
149
+ "content": {
150
+ "type": "json_schema",
151
+ "json_schema": {
152
+ "type": "object",
153
+ "properties": {
154
+ "location": {
155
+ "type": "string"
156
+ }
157
+ },
158
+ "required": [
159
+ "location"
160
+ ]
161
+ },
162
+ "style": "json",
163
+ "any_order": false,
164
+ "max_whitespace_cnt": null,
165
+ "excludes": []
166
+ },
167
+ "end": "}\n</tool_call>"
168
+ },
169
+ {
170
+ "type": "tag",
171
+ "begin": "<tool_call>\n{\"name\": \"add\", \"arguments\": ",
172
+ "content": {
173
+ "type": "json_schema",
174
+ "json_schema": {
175
+ "type": "object",
176
+ "properties": {
177
+ "y": {
178
+ "type": "number"
179
+ },
180
+ "x": {
181
+ "type": "number"
182
+ },
183
+ "items": {
184
+ "type": "array",
185
+ "items": {
186
+ "type": "string"
187
+ }
188
+ }
189
+ },
190
+ "required": [
191
+ "y",
192
+ "x",
193
+ "items"
194
+ ]
195
+ },
196
+ "style": "json",
197
+ "any_order": false,
198
+ "max_whitespace_cnt": null,
199
+ "excludes": []
200
+ },
201
+ "end": "}\n</tool_call>"
202
+ }
203
+ ],
204
+ "at_least_one": true,
205
+ "stop_after_first": false,
206
+ "excludes": [
207
+ "<think>",
208
+ "</think>"
209
+ ]
210
+ }
211
+ ]
212
+ },
213
+ "coverage": "from_token_zero"
214
+ },
215
+ "completion": "<think>\nNeed call the weather and add tools.\n</think>\n\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>\n<tool_call>\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n</tool_call>",
216
+ "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645]
217
+ },
218
+ "named": {
219
+ "grammar": {
220
+ "format": {
221
+ "type": "sequence",
222
+ "elements": [
223
+ {
224
+ "type": "optional",
225
+ "content": {
226
+ "type": "sequence",
227
+ "elements": [
228
+ {
229
+ "type": "tag",
230
+ "begin": "<think>",
231
+ "content": {
232
+ "type": "any_text",
233
+ "excludes": [],
234
+ "max_tokens": null,
235
+ "max_chars": null
236
+ },
237
+ "end": "</think>"
238
+ },
239
+ {
240
+ "type": "const_string",
241
+ "value": "\n\n"
242
+ }
243
+ ]
244
+ }
245
+ },
246
+ {
247
+ "type": "tag",
248
+ "begin": "<tool_call>\n{\"name\": \"get_weather\", \"arguments\": ",
249
+ "content": {
250
+ "type": "json_schema",
251
+ "json_schema": {
252
+ "type": "object",
253
+ "properties": {
254
+ "location": {
255
+ "type": "string"
256
+ }
257
+ },
258
+ "required": [
259
+ "location"
260
+ ]
261
+ },
262
+ "style": "json",
263
+ "any_order": false,
264
+ "max_whitespace_cnt": null,
265
+ "excludes": []
266
+ },
267
+ "end": "}\n</tool_call>"
268
+ }
269
+ ]
270
+ },
271
+ "coverage": "from_token_zero"
272
+ },
273
+ "completion": "<think>\nNeed call the weather and add tools.\n</think>\n\n<tool_call>\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n</tool_call>",
274
+ "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645]
275
+ }
276
+ }
277
+}
rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json
new
+268
@@ -0,0 +1,268 @@
1
+{
2
+ "model_id": "Qwen/Qwen3.5-4B",
3
+ "vocab_size": 248320,
4
+ "all_stop_token_ids": [248044, 248046],
5
+ "cases": {
6
+ "auto": {
7
+ "grammar": {
8
+ "format": {
9
+ "type": "sequence",
10
+ "elements": [
11
+ {
12
+ "type": "sequence",
13
+ "elements": [
14
+ {
15
+ "type": "tag",
16
+ "begin": "",
17
+ "content": {
18
+ "type": "any_text",
19
+ "excludes": [],
20
+ "max_tokens": null,
21
+ "max_chars": null
22
+ },
23
+ "end": "</think>"
24
+ },
25
+ {
26
+ "type": "const_string",
27
+ "value": "\n\n"
28
+ }
29
+ ]
30
+ },
31
+ {
32
+ "type": "triggered_tags",
33
+ "triggers": [
34
+ "<tool_call>\n<function="
35
+ ],
36
+ "tags": [
37
+ {
38
+ "type": "tag",
39
+ "begin": "<tool_call>\n<function=get_weather>\n",
40
+ "content": {
41
+ "type": "json_schema",
42
+ "json_schema": {
43
+ "type": "object",
44
+ "properties": {
45
+ "location": {
46
+ "type": "string"
47
+ }
48
+ },
49
+ "required": [
50
+ "location"
51
+ ]
52
+ },
53
+ "style": "qwen_xml",
54
+ "any_order": false,
55
+ "max_whitespace_cnt": null,
56
+ "excludes": []
57
+ },
58
+ "end": "\n</function>\n</tool_call>"
59
+ },
60
+ {
61
+ "type": "tag",
62
+ "begin": "<tool_call>\n<function=add>\n",
63
+ "content": {
64
+ "type": "json_schema",
65
+ "json_schema": {
66
+ "type": "object",
67
+ "properties": {
68
+ "y": {
69
+ "type": "number"
70
+ },
71
+ "x": {
72
+ "type": "number"
73
+ },
74
+ "items": {
75
+ "type": "array",
76
+ "items": {
77
+ "type": "string"
78
+ }
79
+ }
80
+ },
81
+ "required": [
82
+ "y",
83
+ "x",
84
+ "items"
85
+ ]
86
+ },
87
+ "style": "qwen_xml",
88
+ "any_order": false,
89
+ "max_whitespace_cnt": null,
90
+ "excludes": []
91
+ },
92
+ "end": "\n</function>\n</tool_call>"
93
+ }
94
+ ],
95
+ "at_least_one": false,
96
+ "stop_after_first": false,
97
+ "excludes": [
98
+ "<think>",
99
+ "</think>"
100
+ ]
101
+ }
102
+ ]
103
+ },
104
+ "coverage": "from_token_zero"
105
+ },
106
+ "completion": "Need call the weather and add tools.\n</think>\n\nI will call the tools.\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>\n<tool_call>\n<function=add>\n<parameter=y>\n1.0\n</parameter>\n<parameter=x>\n2\n</parameter>\n<parameter=items>\n[\"left\", \"right\"]\n</parameter>\n</function>\n</tool_call>",
107
+ "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
108
+ },
109
+ "required": {
110
+ "grammar": {
111
+ "format": {
112
+ "type": "sequence",
113
+ "elements": [
114
+ {
115
+ "type": "sequence",
116
+ "elements": [
117
+ {
118
+ "type": "tag",
119
+ "begin": "",
120
+ "content": {
121
+ "type": "any_text",
122
+ "excludes": [],
123
+ "max_tokens": null,
124
+ "max_chars": null
125
+ },
126
+ "end": "</think>"
127
+ },
128
+ {
129
+ "type": "const_string",
130
+ "value": "\n\n"
131
+ }
132
+ ]
133
+ },
134
+ {
135
+ "type": "triggered_tags",
136
+ "triggers": [
137
+ "<tool_call>\n<function="
138
+ ],
139
+ "tags": [
140
+ {
141
+ "type": "tag",
142
+ "begin": "<tool_call>\n<function=get_weather>\n",
143
+ "content": {
144
+ "type": "json_schema",
145
+ "json_schema": {
146
+ "type": "object",
147
+ "properties": {
148
+ "location": {
149
+ "type": "string"
150
+ }
151
+ },
152
+ "required": [
153
+ "location"
154
+ ]
155
+ },
156
+ "style": "qwen_xml",
157
+ "any_order": false,
158
+ "max_whitespace_cnt": null,
159
+ "excludes": []
160
+ },
161
+ "end": "\n</function>\n</tool_call>"
162
+ },
163
+ {
164
+ "type": "tag",
165
+ "begin": "<tool_call>\n<function=add>\n",
166
+ "content": {
167
+ "type": "json_schema",
168
+ "json_schema": {
169
+ "type": "object",
170
+ "properties": {
171
+ "y": {
172
+ "type": "number"
173
+ },
174
+ "x": {
175
+ "type": "number"
176
+ },
177
+ "items": {
178
+ "type": "array",
179
+ "items": {
180
+ "type": "string"
181
+ }
182
+ }
183
+ },
184
+ "required": [
185
+ "y",
186
+ "x",
187
+ "items"
188
+ ]
189
+ },
190
+ "style": "qwen_xml",
191
+ "any_order": false,
192
+ "max_whitespace_cnt": null,
193
+ "excludes": []
194
+ },
195
+ "end": "\n</function>\n</tool_call>"
196
+ }
197
+ ],
198
+ "at_least_one": true,
199
+ "stop_after_first": false,
200
+ "excludes": [
201
+ "<think>",
202
+ "</think>"
203
+ ]
204
+ }
205
+ ]
206
+ },
207
+ "coverage": "from_token_zero"
208
+ },
209
+ "completion": "Need call the weather and add tools.\n</think>\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>\n<tool_call>\n<function=add>\n<parameter=y>\n1.0\n</parameter>\n<parameter=x>\n2\n</parameter>\n<parameter=items>\n[\"left\", \"right\"]\n</parameter>\n</function>\n</tool_call>",
210
+ "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
211
+ },
212
+ "named": {
213
+ "grammar": {
214
+ "format": {
215
+ "type": "sequence",
216
+ "elements": [
217
+ {
218
+ "type": "sequence",
219
+ "elements": [
220
+ {
221
+ "type": "tag",
222
+ "begin": "",
223
+ "content": {
224
+ "type": "any_text",
225
+ "excludes": [],
226
+ "max_tokens": null,
227
+ "max_chars": null
228
+ },
229
+ "end": "</think>"
230
+ },
231
+ {
232
+ "type": "const_string",
233
+ "value": "\n\n"
234
+ }
235
+ ]
236
+ },
237
+ {
238
+ "type": "tag",
239
+ "begin": "<tool_call>\n<function=get_weather>\n",
240
+ "content": {
241
+ "type": "json_schema",
242
+ "json_schema": {
243
+ "type": "object",
244
+ "properties": {
245
+ "location": {
246
+ "type": "string"
247
+ }
248
+ },
249
+ "required": [
250
+ "location"
251
+ ]
252
+ },
253
+ "style": "qwen_xml",
254
+ "any_order": false,
255
+ "max_whitespace_cnt": null,
256
+ "excludes": []
257
+ },
258
+ "end": "\n</function>\n</tool_call>"
259
+ }
260
+ ]
261
+ },
262
+ "coverage": "from_token_zero"
263
+ },
264
+ "completion": "Need call the weather and add tools.\n</think>\n\n<tool_call>\n<function=get_weather>\n<parameter=location>\nShanghai\n</parameter>\n</function>\n</tool_call>",
265
+ "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046]
266
+ }
267
+ }
268
+}
rust/src/chat/tests/roundtrip.rs
+368
-90
@@ -6,20 +6,31 @@
6
//! The invariant under test is that a structured assistant message rendered as history can be
7
//! parsed from the generated assistant completion and then rendered back to the exact same
8
//! assistant completion.
9
-
9
+//!
10
+//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each
11
+//! tool-choice variant, with the rendered completion as its generation, as one file per model under
12
+//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar.
13
+
14
+use std::collections::{BTreeMap, BTreeSet};
15
+use std::fmt::Debug;
16
+use std::ops::Range;
17
+use std::path::PathBuf;
18
use std::pin::Pin;
19
use std::sync::Arc;
20
21
use anyhow::{Context as _, Result, bail, ensure};
22
use futures::{Stream, StreamExt as _, stream};
23
+use serde::ser::Error as _;
24
+use serde::{Serialize, Serializer};
25
use serde_json_fmt::JsonFormat as JsonFmt;
26
use serial_test::file_serial;
27
use vllm_chat::{
28
AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
19
- ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode,
20
- LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection,
21
- load_model_backends,
29
+ ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
30
+ GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
31
+ RendererSelection, ToolStrictLevel, load_model_backends,
32
};
33
+use vllm_parser::output_grammar::BuiltOutputGrammar;
34
use vllm_text::{DecodedTextEvent, Finished, Prompt};
35
use vllm_tokenizer::Tokenizer;
36
@@ -114,6 +125,33 @@ impl ThinkingBehavior {
125
}
126
}
127
128
+/// Tool-choice variant of the tool-call fixture.
129
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)]
130
+#[serde(rename_all = "snake_case")]
131
+enum ToolChoiceVariant {
132
+ /// `auto`, with content before the tool calls.
133
+ Auto,
134
+ /// `required`, with tool calls only: builder grammars admit no content
135
+ /// before the calls.
136
+ Required,
137
+ /// Named `get_weather`, with a single call to it and no content.
138
+ Named,
139
+}
140
+
141
+impl ToolChoiceVariant {
142
+ const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named];
143
+
144
+ fn tool_choice(self) -> ChatToolChoice {
145
+ match self {
146
+ Self::Auto => ChatToolChoice::Auto,
147
+ Self::Required => ChatToolChoice::Required,
148
+ Self::Named => ChatToolChoice::Function {
149
+ name: "get_weather".to_string(),
150
+ },
151
+ }
152
+ }
153
+}
154
+
155
impl RoundtripCase {
156
/// Qwen3 XML tool-call format with `qwen3` reasoning tags.
157
fn qwen3() -> Self {
@@ -451,6 +489,7 @@ async fn run_roundtrip_reasoning_and_content_inner(
489
"roundtrip-reasoning-content",
490
vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
491
Vec::new(),
492
+ None,
493
thinking,
494
case.thinking_behavior,
495
);
@@ -489,11 +528,25 @@ async fn run_roundtrip_reasoning_and_content_inner(
528
Ok(())
529
}
530
492
-/// Run the fixed reasoning+multiple-tools fixture for one model/parser case.
531
+/// Run the fixed reasoning+multiple-tools fixture for one model/parser case,
532
+/// once per tool-choice variant.
533
async fn run_roundtrip_tool_call_mix(
534
case: &RoundtripCase,
535
backends: &vllm_chat::LoadedModelBackends,
536
) -> Result<()> {
537
+ let mut results = Vec::new();
538
+ for variant in ToolChoiceVariant::ALL {
539
+ let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?;
540
+ results.push((variant, result));
541
+ }
542
+ check_grammar_replay_file(case, backends, results)
543
+}
544
+
545
+async fn run_roundtrip_tool_call_mix_inner(
546
+ case: &RoundtripCase,
547
+ backends: &vllm_chat::LoadedModelBackends,
548
+ variant: ToolChoiceVariant,
549
+) -> Result<RoundtripResult> {
550
let request = roundtrip_request(
551
"roundtrip-reasoning-tools",
552
vec![ChatMessage::text(
@@ -501,41 +554,40 @@ async fn run_roundtrip_tool_call_mix(
554
"Check Shanghai weather and add 1.0 plus 2.",
555
)],
556
test_tools(),
557
+ Some(variant.tool_choice()),
558
Some(true), // always enable thinking in this fixture
559
case.thinking_behavior,
560
);
561
let expected_reasoning = "Need call the weather and add tools.";
508
- let expected_text = "I will call the tools.";
509
-
510
- let result = run_roundtrip(
511
- case,
512
- backends,
513
- &request,
514
- AssistantMessage {
515
- content: vec![
516
- AssistantContentBlock::Reasoning {
517
- text: expected_reasoning.to_string(),
518
- },
519
- AssistantContentBlock::Text {
520
- text: expected_text.to_string(),
521
- },
522
- AssistantContentBlock::ToolCall(AssistantToolCall {
523
- id: "functions.get_weather:0".to_string(),
524
- name: "get_weather".to_string(),
525
- arguments: r#"{"location":"Shanghai"}"#.to_string(),
526
- }),
527
- AssistantContentBlock::ToolCall(AssistantToolCall {
528
- id: "functions.add:1".to_string(),
529
- name: "add".to_string(),
530
- // Intentionally use a non-lexical order of keys to verify text-level
531
- // fidelity of the roundtrip where JSON formatting remains stable. The
532
- // `items` key also exercises templates that call `arguments.items()`.
533
- arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
534
- }),
535
- ],
536
- },
537
- )
538
- .await?;
562
+ let (expected_text, tool_call_count) = match variant {
563
+ ToolChoiceVariant::Auto => (Some("I will call the tools."), 2),
564
+ ToolChoiceVariant::Required => (None, 2),
565
+ ToolChoiceVariant::Named => (None, 1),
566
+ };
567
+
568
+ let mut content = vec![AssistantContentBlock::Reasoning {
569
+ text: expected_reasoning.to_string(),
570
+ }];
571
+ content.extend(expected_text.map(|text| AssistantContentBlock::Text {
572
+ text: text.to_string(),
573
+ }));
574
+ let tool_calls = [
575
+ AssistantContentBlock::ToolCall(AssistantToolCall {
576
+ id: "functions.get_weather:0".to_string(),
577
+ name: "get_weather".to_string(),
578
+ arguments: r#"{"location":"Shanghai"}"#.to_string(),
579
+ }),
580
+ AssistantContentBlock::ToolCall(AssistantToolCall {
581
+ id: "functions.add:1".to_string(),
582
+ name: "add".to_string(),
583
+ // Intentionally use a non-lexical order of keys to verify text-level
584
+ // fidelity of the roundtrip where JSON formatting remains stable. The
585
+ // `items` key also exercises templates that call `arguments.items()`.
586
+ arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
587
+ }),
588
+ ];
589
+ content.extend(tool_calls.into_iter().take(tool_call_count));
590
+ let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?;
591
592
assert_eq!(
593
result.parsed_message.reasoning().as_deref(),
@@ -543,12 +595,15 @@ async fn run_roundtrip_tool_call_mix(
595
"parsed message: {:#?}",
596
result.parsed_message
597
);
546
- assert_eq!(result.parsed_message.text(), expected_text);
598
+ assert_eq!(
599
+ result.parsed_message.text(),
600
+ expected_text.unwrap_or_default()
601
+ );
602
603
let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
604
assert_eq!(
605
tool_calls.len(),
551
- 2,
606
+ tool_call_count,
607
"parsed message: {:#?}",
608
result.parsed_message
609
);
@@ -557,18 +612,20 @@ async fn run_roundtrip_tool_call_mix(
612
tool_calls[0].arguments,
613
expected_arguments(case, r#"{"location": "Shanghai"}"#)?,
614
);
560
- assert_eq!(tool_calls[1].name, "add");
561
- assert_eq!(
562
- tool_calls[1].arguments,
563
- expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
564
- );
615
+ if let Some(add) = tool_calls.get(1) {
616
+ assert_eq!(add.name, "add");
617
+ assert_eq!(
618
+ add.arguments,
619
+ expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
620
+ );
621
+ }
622
623
assert_eq!(
624
result.rerendered_closed_completion,
625
result.closed_completion
626
);
627
571
- Ok(())
628
+ Ok(result)
629
}
630
631
/// Compact JSON argument formatting used by JSON-native parsers/renderers.
@@ -645,6 +702,9 @@ struct RoundtripResult {
702
/// Assistant-completion suffix cut after rendering the parsed assistant
703
/// back as history.
704
rerendered_closed_completion: Prompt,
705
+ /// Output grammar built by the same initialized parser that parsed the
706
+ /// completion.
707
+ output_grammar: Option<BuiltOutputGrammar>,
708
}
709
710
/// Render, parse, and rerender one assistant turn through the production
@@ -658,13 +718,17 @@ async fn run_roundtrip(
718
let renderer = backends.chat_backend.chat_renderer();
719
let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?;
720
661
- let parsed_message = parse_completion(case, backends, request, &rendered).await?;
721
+ let ParsedCompletion {
722
+ message: parsed_message,
723
+ output_grammar,
724
+ } = parse_completion(case, backends, request, &rendered).await?;
725
let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
726
727
Ok(RoundtripResult {
728
parsed_message,
729
closed_completion: rendered.completion,
730
rerendered_closed_completion: rerendered.completion,
731
+ output_grammar,
732
})
733
}
734
@@ -675,7 +739,7 @@ struct RenderedTurn {
739
}
740
741
/// Render `history` as a production prompt and `history + assistant` as closed
678
-/// history, then return the production prompt and assistant-completion suffix.
742
+/// history, then return the production prompt and the assistant completion.
743
fn render_closed_completion(
744
renderer: &dyn vllm_chat::ChatRenderer,
745
base_request: &ChatRequest,
@@ -692,18 +756,15 @@ fn render_closed_completion(
756
757
let completion = match (&prompt, full) {
758
(Prompt::Text(prompt), Prompt::Text(full)) => {
695
- ensure!(
696
- full.starts_with(prompt),
697
- "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
698
- );
699
- Prompt::Text(full[prompt.len()..].to_string())
759
+ let range = completion_range(prompt.as_bytes(), full.as_bytes())
760
+ .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?;
761
+ let completion = full.get(range).context("completion splits a UTF-8 character")?;
762
+ Prompt::Text(completion.to_string())
763
}
764
(Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => {
702
- ensure!(
703
- full.starts_with(prompt),
704
- "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
705
- );
706
- Prompt::TokenIds(full[prompt.len()..].to_vec())
765
+ let range = completion_range(prompt, &full)
766
+ .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?;
767
+ Prompt::TokenIds(full[range].to_vec())
768
}
769
(prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"),
770
};
@@ -711,14 +772,81 @@ fn render_closed_completion(
772
Ok(RenderedTurn { prompt, completion })
773
}
774
775
+/// Locate the assistant completion in `full`, the history rendered with the
776
+/// assistant message, given `prompt`, the same history rendered for generation.
777
+///
778
+/// Some templates render request-scoped control messages after the whole
779
+/// history, such as Kimi K3's `tool_choice=required` message. With history `A`,
780
+/// control messages `C`, generation prompt `G`, and completion `B`, the renders
781
+/// are `prompt = A + C + G` and `full = A + G + B + C`. `C` is empty for most
782
+/// templates, which makes `prompt` a prefix of `full`.
783
+fn completion_range<T: PartialEq + Debug>(prompt: &[T], full: &[T]) -> Result<Range<usize>> {
784
+ if full.starts_with(prompt) {
785
+ return Ok(prompt.len()..full.len());
786
+ }
787
+ ensure!(
788
+ full.len() >= prompt.len(),
789
+ "full prompt is shorter than production prompt"
790
+ );
791
+
792
+ // `A` is a common prefix of both renders; try every split of the remaining
793
+ // `prompt` into `C + G` and require exactly one to fit `full`.
794
+ let common_prefix_len = prompt.iter().zip(full).take_while(|(p, f)| p == f).count();
795
+ let mut control_lens = Vec::new();
796
+ for history_len in 0..=common_prefix_len {
797
+ let rest = &prompt[history_len..];
798
+ for control_len in 1..=rest.len() {
799
+ let (control, generation) = rest.split_at(control_len);
800
+ if full[history_len..].starts_with(generation) && full.ends_with(control) {
801
+ control_lens.push(control_len);
802
+ }
803
+ }
804
+ }
805
+ let [control_len] = control_lens[..] else {
806
+ bail!(
807
+ "full prompt must be `A + G + B + C` for production prompt `A + C + G`; found {} splits",
808
+ control_lens.len()
809
+ );
810
+ };
811
+
812
+ Ok(prompt.len() - control_len..full.len() - control_len)
813
+}
814
+
815
+#[test]
816
+fn completion_range_skips_trailing_control_messages() {
817
+ let history = "<user>hi</user>";
818
+ let control = "<system>call tools</system>";
819
+ let generation = "<assistant>";
820
+ let completion = "call()</assistant>";
821
+
822
+ let prompt = format!("{history}{generation}");
823
+ let full = format!("{history}{generation}{completion}");
824
+ let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap();
825
+ assert_eq!(&full[range], completion);
826
+
827
+ let prompt = format!("{history}{control}{generation}");
828
+ let full = format!("{history}{generation}{completion}{control}");
829
+ let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap();
830
+ assert_eq!(&full[range], completion);
831
+}
832
+
833
+/// Output of one production output processor over a rendered completion.
834
+struct ParsedCompletion {
835
+ /// Terminal assistant message.
836
+ message: AssistantMessage,
837
+ /// Output grammar built by the processor after initialization.
838
+ output_grammar: Option<BuiltOutputGrammar>,
839
+}
840
+
841
/// Feed one rendered assistant completion body into the real output processor
715
-/// and collect its terminal assistant message.
842
+/// and collect its terminal assistant message, along with the output grammar
843
+/// the initialized processor builds for the request.
844
async fn parse_completion(
845
case: &RoundtripCase,
846
backends: &vllm_chat::LoadedModelBackends,
847
base_request: &ChatRequest,
848
rendered: &RenderedTurn,
721
-) -> Result<AssistantMessage> {
849
+) -> Result<ParsedCompletion> {
850
let tokenizer = backends.text_backend.tokenizer();
851
let prompt_token_ids = match &rendered.prompt {
852
Prompt::Text(prompt) => tokenizer
@@ -731,12 +859,15 @@ async fn parse_completion(
859
let mut processor = backends.chat_backend.new_chat_output_processor(
860
&mut request,
861
NewChatOutputProcessorOptions {
734
- tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
862
+ // Pin every tool schema so that builder-backed parsers build an
863
+ // output grammar for replay. The strict level does not affect parsing.
864
+ tool_strict_level: ToolStrictLevel::Parameter,
865
tool_call_parser: &case.tool_call_parser,
866
reasoning_parser: &case.reasoning_parser,
867
},
868
)?;
869
processor.initialize(&prompt_token_ids)?;
870
+ let output_grammar = processor.build_output_grammar()?;
871
872
let decoded = decoded_completion_stream(
873
tokenizer.as_ref(),
@@ -748,7 +879,10 @@ async fn parse_completion(
879
880
while let Some(event) = events.next().await {
881
if let ChatEvent::Done { message, .. } = event? {
751
- return Ok(message);
882
+ return Ok(ParsedCompletion {
883
+ message,
884
+ output_grammar,
885
+ });
886
}
887
}
888
@@ -772,14 +906,8 @@ fn decoded_completion_stream(
906
prompt_logprobs: None,
907
}];
908
775
- let chunks = match completion {
776
- Prompt::Text(text) => {
777
- let body = text.strip_suffix(assistant_stop_suffix).with_context(|| {
778
- format!(
779
- "closed assistant completion did not end with {:?}: {:?}",
780
- assistant_stop_suffix, text
781
- )
782
- })?;
909
+ let chunks = match completion_body(tokenizer, completion, assistant_stop_suffix)? {
910
+ CompletionBody::Text(body) => {
911
// The engine returns token ids, so attribute the rendered text
912
// through the production decoder, then split it into small char
913
// chunks to exercise marker splits.
@@ -807,27 +935,12 @@ fn decoded_completion_stream(
935
})
936
.collect()
937
}
810
- Prompt::TokenIds(token_ids) => {
811
- let body = if assistant_stop_suffix.is_empty() {
812
- token_ids.as_slice()
813
- } else {
814
- let stop_token_ids = tokenizer
815
- .encode(assistant_stop_suffix, false)
816
- .context("failed to encode token-id completion stop suffix")?;
817
- token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
818
- format!(
819
- "token-id completion did not end with {:?}: {:?}",
820
- assistant_stop_suffix, token_ids
821
- )
822
- })?
823
- };
824
- incremental_decode_chunks(
825
- tokenizer,
826
- &prompt_token_ids,
827
- body,
828
- TOKEN_COMPLETION_CHUNK_TOKENS,
829
- )?
830
- }
938
+ CompletionBody::TokenIds(body) => incremental_decode_chunks(
939
+ tokenizer,
940
+ &prompt_token_ids,
941
+ body,
942
+ TOKEN_COMPLETION_CHUNK_TOKENS,
943
+ )?,
944
};
945
946
if chunks.is_empty() {
@@ -868,6 +981,48 @@ fn decoded_completion_stream(
981
Ok(stream::iter(events).map(Ok).boxed())
982
}
983
984
+/// Generated part of a closed assistant completion, without the rendered stop
985
+/// suffix.
986
+enum CompletionBody<'a> {
987
+ Text(&'a str),
988
+ TokenIds(&'a [u32]),
989
+}
990
+
991
+/// Strip the rendered stop suffix from a closed assistant completion.
992
+fn completion_body<'a>(
993
+ tokenizer: &dyn Tokenizer,
994
+ completion: &'a Prompt,
995
+ assistant_stop_suffix: &str,
996
+) -> Result<CompletionBody<'a>> {
997
+ match completion {
998
+ Prompt::Text(text) => {
999
+ let body = text.strip_suffix(assistant_stop_suffix).with_context(|| {
1000
+ format!(
1001
+ "closed assistant completion did not end with {:?}: {:?}",
1002
+ assistant_stop_suffix, text
1003
+ )
1004
+ })?;
1005
+ Ok(CompletionBody::Text(body))
1006
+ }
1007
+ Prompt::TokenIds(token_ids) => {
1008
+ let body = if assistant_stop_suffix.is_empty() {
1009
+ token_ids.as_slice()
1010
+ } else {
1011
+ let stop_token_ids = tokenizer
1012
+ .encode(assistant_stop_suffix, false)
1013
+ .context("failed to encode token-id completion stop suffix")?;
1014
+ token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
1015
+ format!(
1016
+ "token-id completion did not end with {:?}: {:?}",
1017
+ assistant_stop_suffix, token_ids
1018
+ )
1019
+ })?
1020
+ };
1021
+ Ok(CompletionBody::TokenIds(body))
1022
+ }
1023
+ }
1024
+}
1025
+
1026
/// One decoded completion chunk fed into the output processor.
1027
struct DecodedCompletionChunk {
1028
/// Attributed for token-id completions, as the production decoder emits it.
@@ -941,15 +1096,138 @@ fn incremental_decode_chunks(
1096
Ok(chunks)
1097
}
1098
1099
+/// Grammar replay cases of one model, replayed through the real XGrammar by
1100
+/// `grammar_replay.py`.
1101
+#[derive(Serialize)]
1102
+struct GrammarReplayFile {
1103
+ /// Hugging Face model id whose tokenizer the replay loads.
1104
+ model_id: &'static str,
1105
+ /// Model vocabulary size, which sizes the XGrammar token mask.
1106
+ vocab_size: usize,
1107
+ /// Request stop set, passed to the matcher as `override_stop_tokens`.
1108
+ #[serde(serialize_with = "serialize_one_line")]
1109
+ all_stop_token_ids: BTreeSet<u32>,
1110
+ /// Cases keyed by tool-choice variant.
1111
+ cases: BTreeMap<ToolChoiceVariant, GrammarReplayCase>,
1112
+}
1113
+
1114
+/// One parser-built output grammar with the generation it must accept.
1115
+#[derive(Serialize)]
1116
+struct GrammarReplayCase {
1117
+ /// Parser-built grammar; only token-zero grammars are replayed from the
1118
+ /// first generated token.
1119
+ grammar: BuiltOutputGrammar,
1120
+ /// Generated completion text, without the stop token.
1121
+ completion: String,
1122
+ /// Generated token IDs presented to the matcher, ending with a stop token.
1123
+ #[serde(serialize_with = "serialize_one_line")]
1124
+ generation_token_ids: Vec<u32>,
1125
+}
1126
+
1127
+/// Serialize a token ID list on one line inside the pretty-printed file.
1128
+fn serialize_one_line<S: Serializer>(
1129
+ token_ids: &impl Serialize,
1130
+ serializer: S,
1131
+) -> std::result::Result<S::Ok, S::Error> {
1132
+ let json = spaced_json_fmt().format_to_string(token_ids).map_err(S::Error::custom)?;
1133
+ serde_json::value::RawValue::from_string(json)
1134
+ .map_err(S::Error::custom)?
1135
+ .serialize(serializer)
1136
+}
1137
+
1138
+/// Directory of checked-in grammar replay files.
1139
+fn grammar_replay_dir() -> PathBuf {
1140
+ PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay")
1141
+}
1142
+
1143
+/// Snapshot the output grammars of one model's roundtrips, keyed by variant,
1144
+/// as its grammar replay file, or check that no stale file remains when the
1145
+/// parser builds none.
1146
+fn check_grammar_replay_file(
1147
+ case: &RoundtripCase,
1148
+ backends: &vllm_chat::LoadedModelBackends,
1149
+ results: Vec<(ToolChoiceVariant, RoundtripResult)>,
1150
+) -> Result<()> {
1151
+ let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name);
1152
+ let path = grammar_replay_dir().join(format!("{model_name}.json"));
1153
+ let results = results
1154
+ .into_iter()
1155
+ .filter_map(|(variant, result)| {
1156
+ Some((variant, result.output_grammar?, result.closed_completion))
1157
+ })
1158
+ .collect::<Vec<_>>();
1159
+ if results.is_empty() {
1160
+ ensure!(
1161
+ !path.exists(),
1162
+ "{} builds no output grammar, but grammar replay file {path:?} exists",
1163
+ case.model_id
1164
+ );
1165
+ return Ok(());
1166
+ }
1167
+
1168
+ let tokenizer = backends.text_backend.tokenizer();
1169
+ let hints = backends.text_backend.sampling_hints()?;
1170
+ // Lowered `_all_stop_token_ids` of a request without user stop tokens.
1171
+ let mut all_stop_token_ids = hints.extra_eos_token_ids;
1172
+ all_stop_token_ids.extend(hints.primary_eos_token_id);
1173
+
1174
+ // The generation ends with the first token of the rendered stop suffix, or
1175
+ // with the primary EOS for templates that render none.
1176
+ let stop_token_id = if case.assistant_stop_suffix.is_empty() {
1177
+ hints.primary_eos_token_id.context("model has no primary EOS token")?
1178
+ } else {
1179
+ tokenizer.encode(case.assistant_stop_suffix, false)?[0]
1180
+ };
1181
+ ensure!(
1182
+ all_stop_token_ids.contains(&stop_token_id),
1183
+ "stop token {stop_token_id} of {:?} is not in the stop set {all_stop_token_ids:?}",
1184
+ case.assistant_stop_suffix
1185
+ );
1186
+
1187
+ let mut cases = BTreeMap::new();
1188
+ for (variant, grammar, closed_completion) in results {
1189
+ let (completion, mut generation_token_ids) = match completion_body(
1190
+ tokenizer.as_ref(),
1191
+ &closed_completion,
1192
+ case.assistant_stop_suffix,
1193
+ )? {
1194
+ CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?),
1195
+ CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()),
1196
+ };
1197
+ generation_token_ids.push(stop_token_id);
1198
+ cases.insert(
1199
+ variant,
1200
+ GrammarReplayCase {
1201
+ grammar,
1202
+ completion,
1203
+ generation_token_ids,
1204
+ },
1205
+ );
1206
+ }
1207
+
1208
+ let file = GrammarReplayFile {
1209
+ model_id: case.model_id,
1210
+ vocab_size: backends.text_backend.model_vocab_size(),
1211
+ all_stop_token_ids,
1212
+ cases,
1213
+ };
1214
+ let mut json = serde_json::to_string_pretty(&file)?;
1215
+ json.push('\n');
1216
+ expect_test::expect_file![path].assert_eq(&json);
1217
+
1218
+ Ok(())
1219
+}
1220
+
1221
/// Build a chat request fixture with parser-enabling tool-choice semantics.
1222
fn roundtrip_request(
1223
request_id: impl Into<String>,
1224
messages: Vec<ChatMessage>,
1225
tools: Vec<ChatTool>,
1226
+ tool_choice: Option<ChatToolChoice>,
1227
thinking: Option<bool>,
1228
thinking_behavior: ThinkingBehavior,
1229
) -> ChatRequest {
952
- let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
1230
+ let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, tool_choice, true)
1231
.expect("tool context should resolve");
1232
let mut request = ChatRequest {
1233
request_id: request_id.into(),
rust/src/parser/src/output_grammar.rs
+4
-2
@@ -5,6 +5,7 @@
5
6
use std::{fmt, str::FromStr};
7
8
+use serde::Serialize;
9
use serde_with::{DeserializeFromStr, SerializeDisplay};
10
use thiserror::Error;
11
use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions};
@@ -20,7 +21,8 @@ use crate::tool::Tool;
21
pub type Result<T> = std::result::Result<T, OutputGrammarError>;
22
23
/// How much of the generated stream an output grammar covers.
23
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
25
+#[serde(rename_all = "snake_case")]
26
pub enum GrammarCoverage {
27
/// The grammar covers reasoning and advances from the first generated token.
28
FromTokenZero,
@@ -29,7 +31,7 @@ pub enum GrammarCoverage {
31
}
32
33
/// One parser-built output grammar and its generated-stream coverage.
32
-#[derive(Debug, Clone, PartialEq)]
34
+#[derive(Debug, Clone, PartialEq, Serialize)]
35
pub struct BuiltOutputGrammar {
36
/// Typed XGrammar structural-tag format.
37
pub format: Format,