A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 02:15 UTC
[Frontend] supports deepseekv32 chat template (#29837)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Dec 3, 2025 at 20:53 UTC
b78772c433515a22bfeeaea41f3524002609e264
5 files changed
+616
-2
vllm/config/model.py
+2
-1
@@ -84,7 +84,7 @@ TaskOption = Literal[
84
"transcription",
85
"draft",
86
]
87
-TokenizerMode = Literal["auto", "hf", "slow", "mistral"]
87
+TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32"]
88
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
89
LogprobsMode = Literal[
90
"raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -141,6 +141,7 @@ class ModelConfig:
141
- "hf" will use the fast tokenizer if available.\n
142
- "slow" will always use the slow tokenizer.\n
143
- "mistral" will always use the tokenizer from `mistral_common`.\n
144
+ - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
145
- Other custom values can be supported via plugins."""
146
trust_remote_code: bool = False
147
"""Trust remote code (e.g., from HuggingFace) when downloading the model
vllm/entrypoints/openai/serving_engine.py
+8
-1
@@ -105,7 +105,7 @@ from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
105
from vllm.pooling_params import PoolingParams
106
from vllm.reasoning import ReasoningParser, ReasoningParserManager
107
from vllm.sampling_params import BeamSearchParams, SamplingParams
108
-from vllm.tokenizers import MistralTokenizer, TokenizerLike
108
+from vllm.tokenizers import DeepseekV32Tokenizer, MistralTokenizer, TokenizerLike
109
from vllm.tracing import (
110
contains_trace_headers,
111
extract_trace_headers,
@@ -1128,6 +1128,13 @@ class OpenAIServing:
1128
messages=messages,
1129
**_chat_template_kwargs,
1130
)
1131
+ elif isinstance(tokenizer, DeepseekV32Tokenizer):
1132
+ request_prompt = tokenizer.apply_chat_template(
1133
+ conversation=conversation,
1134
+ messages=messages,
1135
+ model_config=model_config,
1136
+ **_chat_template_kwargs,
1137
+ )
1138
else:
1139
request_prompt = apply_hf_chat_template(
1140
tokenizer=tokenizer,
vllm/tokenizers/__init__.py
+2
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from .deepseekv32 import DeepseekV32Tokenizer
5
from .hf import HfTokenizer
6
from .mistral import MistralTokenizer
7
from .protocol import TokenizerLike
@@ -21,4 +22,5 @@ __all__ = [
22
"get_tokenizer",
23
"cached_tokenizer_from_config",
24
"init_tokenizer_from_config",
25
+ "DeepseekV32Tokenizer",
26
]
vllm/tokenizers/deepseek_v32_encoding.py
new
+456
@@ -0,0 +1,456 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+# copy from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py
6
+import copy
7
+import json
8
+import re
9
+from typing import Any
10
+
11
+# flake8: noqa: E501
12
+TOOLS_SYSTEM_TEMPLATE = """## Tools
13
+You have access to a set of tools you can use to answer the user's question.
14
+You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user:
15
+<{dsml_token}function_calls>
16
+<{dsml_token}invoke name="$FUNCTION_NAME">
17
+<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}parameter>
18
+...
19
+</{dsml_token}invoke>
20
+<{dsml_token}invoke name="$FUNCTION_NAME2">
21
+...
22
+</{dsml_token}invoke>
23
+</{dsml_token}function_calls>
24
+String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects).
25
+If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example:
26
+<{dsml_token}function_calls>
27
+...
28
+</{dsml_token}function_calls>
29
+<function_results>
30
+...
31
+</function_results>
32
+{thinking_start_token}...thinking about results{thinking_end_token}
33
+Here are the functions available in JSONSchema format:
34
+<functions>
35
+{tool_schemas}
36
+</functions>
37
+"""
38
+
39
+bos_token: str = "<｜begin▁of▁sentence｜>"
40
+eos_token: str = "<｜end▁of▁sentence｜>"
41
+thinking_start_token: str = "<think>"
42
+thinking_end_token: str = "</think>"
43
+dsml_token: str = "｜DSML｜"
44
+system_msg_template: str = "{content}"
45
+user_msg_template: str = "<｜User｜>{content}<｜Assistant｜>"
46
+assistant_msg_template: str = "{reasoning}{content}{tool_calls}<｜end▁of▁sentence｜>"
47
+thinking_template = "{reasoning_content}"
48
+
49
+response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"
50
+tool_call_template: str = (
51
+ '<{dsml_token}invoke name="{name}">\n{arguments}\n</{dsml_token}invoke>'
52
+)
53
+tool_calls_template = (
54
+ "<{dsml_token}function_calls>\n{tool_calls}\n</{dsml_token}function_calls>"
55
+)
56
+
57
+tool_output_template: str = "\n<result>{content}</result>"
58
+
59
+
60
+def to_json(value: Any) -> str:
61
+ try:
62
+ return json.dumps(value, ensure_ascii=False)
63
+ except Exception:
64
+ return json.dumps(value, ensure_ascii=True)
65
+
66
+
67
+def tools_from_openai_format(tools):
68
+ return [tool["function"] for tool in tools]
69
+
70
+
71
+def tool_calls_from_openai_format(tool_calls):
72
+ return [
73
+ {
74
+ "name": tool_call["function"]["name"],
75
+ "arguments": tool_call["function"]["arguments"],
76
+ }
77
+ for tool_call in tool_calls
78
+ ]
79
+
80
+
81
+def tool_calls_to_openai_format(tool_calls):
82
+ return [
83
+ {
84
+ "type": "function",
85
+ "function": {
86
+ "name": tool_call["name"],
87
+ "arguments": tool_call["arguments"],
88
+ },
89
+ }
90
+ for tool_call in tool_calls
91
+ ]
92
+
93
+
94
+def encode_arguments_to_dsml(tool_call: dict[str, str]) -> str:
95
+ p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}</{dsml_token}parameter>"""
96
+ P_dsml_strs = []
97
+
98
+ arguments = json.loads(tool_call["arguments"])
99
+
100
+ for k, v in arguments.items():
101
+ p_dsml_str = p_dsml_template.format(
102
+ dsml_token=dsml_token,
103
+ key=k,
104
+ is_str="true" if isinstance(v, str) else "false",
105
+ value=v if isinstance(v, str) else to_json(v),
106
+ )
107
+
108
+ P_dsml_strs.append(p_dsml_str)
109
+
110
+ return "\n".join(P_dsml_strs)
111
+
112
+
113
+def decode_dsml_to_arguments(
114
+ tool_name: str, tool_args: dict[str, tuple[str, str]]
115
+) -> dict[str, str]:
116
+ def _decode_value(key: str, value: str, string: str):
117
+ if string == "true":
118
+ value = to_json(value)
119
+ return f"{to_json(key)}: {value}"
120
+
121
+ tool_args_json = (
122
+ "{"
123
+ + ", ".join(
124
+ [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]
125
+ )
126
+ + "}"
127
+ )
128
+ return dict(name=tool_name, arguments=tool_args_json)
129
+
130
+
131
+def render_tools(tools: list[dict[str, str | dict[str, Any]]]) -> str:
132
+ tools_json = [to_json(t) for t in tools]
133
+
134
+ return TOOLS_SYSTEM_TEMPLATE.format(
135
+ tool_schemas="\n".join(tools_json),
136
+ dsml_token=dsml_token,
137
+ thinking_start_token=thinking_start_token,
138
+ thinking_end_token=thinking_end_token,
139
+ )
140
+
141
+
142
+def find_last_user_index(messages: list[dict[str, Any]]) -> int:
143
+ last_user_index = -1
144
+ for idx in range(len(messages) - 1, -1, -1):
145
+ if messages[idx].get("role") in ["user", "developer"]:
146
+ last_user_index = idx
147
+ break
148
+ return last_user_index
149
+
150
+
151
+def render_message(
152
+ index: int, messages: list[dict[str, Any]], thinking_mode: str
153
+) -> str:
154
+ assert 0 <= index < len(messages)
155
+ assert thinking_mode in ["chat", "thinking"], (
156
+ f"Invalid thinking_mode `{thinking_mode}`"
157
+ )
158
+
159
+ prompt = ""
160
+ msg = messages[index]
161
+ last_user_idx = find_last_user_index(messages)
162
+
163
+ role = msg.get("role")
164
+ content = msg.get("content")
165
+ tools = msg.get("tools")
166
+ response_format = msg.get("response_format")
167
+ tool_calls = msg.get("tool_calls")
168
+ reasoning_content = msg.get("reasoning") or msg.get("reasoning_content")
169
+
170
+ if tools:
171
+ tools = tools_from_openai_format(tools)
172
+ if tool_calls:
173
+ tool_calls = tool_calls_from_openai_format(tool_calls)
174
+
175
+ if role == "system":
176
+ prompt += system_msg_template.format(content=content or "")
177
+ if tools:
178
+ prompt += "\n\n" + render_tools(tools)
179
+
180
+ if response_format:
181
+ prompt += "\n\n" + response_format_template.format(
182
+ schema=to_json(response_format)
183
+ )
184
+
185
+ elif role == "developer":
186
+ assert content, f"Invalid message for role `{role}`: {msg}"
187
+ content_developer = ""
188
+ if tools:
189
+ content_developer += "\n\n" + render_tools(tools)
190
+
191
+ if response_format:
192
+ content_developer += "\n\n" + response_format_template.format(
193
+ schema=to_json(response_format)
194
+ )
195
+
196
+ content_developer += "\n\n# The user's message is: {}".format(content)
197
+
198
+ prompt += user_msg_template.format(content=content_developer)
199
+ if index == last_user_idx and thinking_mode == "thinking":
200
+ prompt += thinking_start_token
201
+ else:
202
+ prompt += thinking_end_token
203
+
204
+ elif role == "user":
205
+ prompt += user_msg_template.format(content=content)
206
+
207
+ if index == last_user_idx and thinking_mode == "thinking":
208
+ prompt += thinking_start_token
209
+ else:
210
+ prompt += thinking_end_token
211
+
212
+ elif role == "tool":
213
+ prev_assistant_idx = index - 1
214
+ assistant_msg = messages[prev_assistant_idx]
215
+ while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool":
216
+ prev_assistant_idx -= 1
217
+ assistant_msg = messages[prev_assistant_idx]
218
+
219
+ assert (
220
+ index == 0
221
+ or prev_assistant_idx >= 0
222
+ and assistant_msg.get("role") == "assistant"
223
+ ), f"Invalid messages at {index}:\n{assistant_msg}"
224
+
225
+ tool_call_order = index - prev_assistant_idx
226
+ assistant_tool_calls = assistant_msg.get("tool_calls")
227
+ assert assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order, (
228
+ "No tool calls but found tool output"
229
+ )
230
+
231
+ if tool_call_order == 1:
232
+ prompt += "\n\n<function_results>"
233
+
234
+ prompt += tool_output_template.format(content=content)
235
+
236
+ if tool_call_order == len(assistant_tool_calls):
237
+ prompt += "\n</function_results>"
238
+
239
+ if index >= last_user_idx and thinking_mode == "thinking":
240
+ prompt += "\n\n" + thinking_start_token
241
+ else:
242
+ prompt += "\n\n" + thinking_end_token
243
+
244
+ elif role == "assistant":
245
+ prev_assistant_idx = index
246
+ thinking_part = ""
247
+
248
+ tool_calls_content = ""
249
+ if tool_calls:
250
+ tool_calls = [
251
+ tool_call_template.format(
252
+ dsml_token=dsml_token,
253
+ name=tool_call.get("name"),
254
+ arguments=encode_arguments_to_dsml(tool_call),
255
+ )
256
+ for tool_call in tool_calls
257
+ ]
258
+ tool_calls_content += "\n\n" + tool_calls_template.format(
259
+ dsml_token=dsml_token, tool_calls="\n".join(tool_calls)
260
+ )
261
+
262
+ summary_content = content or ""
263
+
264
+ if thinking_mode == "thinking" and index > last_user_idx:
265
+ assert reasoning_content or tool_calls, (
266
+ f"ThinkingMode: {thinking_mode}, invalid message without reasoning_content/tool_calls `{msg}` after last user message"
267
+ )
268
+ thinking_part = (
269
+ thinking_template.format(reasoning_content=reasoning_content or "")
270
+ + thinking_end_token
271
+ )
272
+
273
+ prompt += assistant_msg_template.format(
274
+ reasoning=thinking_part,
275
+ content=summary_content,
276
+ tool_calls=tool_calls_content,
277
+ )
278
+ else:
279
+ raise NotImplementedError(f"Unknown role: {role}")
280
+
281
+ return prompt
282
+
283
+
284
+def drop_thinking_messages(
285
+ messages: list[dict[str, Any]], last_user_idx: int | None = None
286
+) -> list[dict[str, Any]]:
287
+ messages_wo_thinking: list[dict[str, Any]] = []
288
+ last_user_idx = (
289
+ find_last_user_index(messages) if last_user_idx is None else last_user_idx
290
+ )
291
+ for idx, msg in enumerate(messages):
292
+ role = msg.get("role")
293
+ if role in ["user", "system", "tool"] or idx >= last_user_idx:
294
+ messages_wo_thinking.append(msg)
295
+ continue
296
+
297
+ elif role == "assistant":
298
+ msg_wo_thinking = copy.copy(msg)
299
+ msg_wo_thinking.pop("reasoning_content", None)
300
+ msg_wo_thinking.pop("reasoning", None)
301
+ messages_wo_thinking.append(msg_wo_thinking)
302
+
303
+ return messages_wo_thinking
304
+
305
+
306
+def encode_messages(
307
+ messages: list[dict[str, Any]],
308
+ thinking_mode: str,
309
+ context: list[dict[str, Any]] | None = None,
310
+ drop_thinking: bool = True,
311
+ add_default_bos_token: bool = True,
312
+) -> str:
313
+ context = context if context else []
314
+ full_messages = context + messages
315
+
316
+ prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
317
+
318
+ if thinking_mode == "thinking" and drop_thinking:
319
+ full_messages = drop_thinking_messages(full_messages)
320
+
321
+ for idx in range(len(messages)):
322
+ prompt += render_message(
323
+ idx + len(context), full_messages, thinking_mode=thinking_mode
324
+ )
325
+
326
+ return prompt
327
+
328
+
329
+def _read_until_stop(
330
+ index: int, text: str, stop: list[str]
331
+) -> tuple[int, str, None | str]:
332
+ min_pos = len(text)
333
+ matched_stop = None
334
+
335
+ for s in stop:
336
+ pos = text.find(s, index)
337
+ if pos != -1 and pos < min_pos:
338
+ min_pos = pos
339
+ matched_stop = s
340
+
341
+ if matched_stop:
342
+ content = text[index:min_pos]
343
+ return min_pos + len(matched_stop), content, matched_stop
344
+ else:
345
+ content = text[index:]
346
+ return len(text), content, None
347
+
348
+
349
+def parse_tool_calls(index: int, text: str):
350
+ tool_calls: list[dict[str, Any]] = []
351
+ stop_token = None
352
+ tool_calls_end_token = f"</{dsml_token}function_calls>"
353
+
354
+ while index < len(text):
355
+ index, _, stop_token = _read_until_stop(
356
+ index, text, [f"<{dsml_token}invoke", tool_calls_end_token]
357
+ )
358
+ assert _ == ">\n", "Tool call format error"
359
+
360
+ if stop_token == tool_calls_end_token:
361
+ break
362
+
363
+ assert stop_token is not None, "Missing special token"
364
+
365
+ index, tool_name_content, stop_token = _read_until_stop(
366
+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
367
+ )
368
+
369
+ p_tool_name = re.findall(
370
+ r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL
371
+ )
372
+ assert len(p_tool_name) == 1, "Tool name format error"
373
+ tool_name = p_tool_name[0]
374
+
375
+ tool_args: dict[str, tuple[str, str]] = {}
376
+ while stop_token == f"<{dsml_token}parameter":
377
+ index, param_content, stop_token = _read_until_stop(
378
+ index, text, [f"/{dsml_token}parameter"]
379
+ )
380
+
381
+ param_kv = re.findall(
382
+ r'^ name="(.*?)" string="(true|false)">(.*?)<$',
383
+ param_content,
384
+ flags=re.DOTALL,
385
+ )
386
+ assert len(param_kv) == 1, "Parameter format error"
387
+ param_name, string, param_value = param_kv[0]
388
+
389
+ assert param_name not in tool_args, "Duplicate parameter name"
390
+ tool_args[param_name] = (param_value, string)
391
+
392
+ index, content, stop_token = _read_until_stop(
393
+ index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
394
+ )
395
+ assert content == ">\n", "Parameter format error"
396
+
397
+ tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args)
398
+ tool_calls.append(tool_call)
399
+
400
+ return index, stop_token, tool_calls
401
+
402
+
403
+# NOTE: This function is designed to parse only correctly
404
+# formatted string and will not attempt to correct malformed output
405
+# that may be generated by the model.
406
+def parse_message_from_completion_text(text: str, thinking_mode: str):
407
+ summary_content, reasoning_content, tool_calls = "", "", []
408
+ index, stop_token = 0, None
409
+ tool_calls_start_token = f"\n\n<{dsml_token}function_calls"
410
+
411
+ is_thinking, is_tool_calling = thinking_mode == "thinking", False
412
+
413
+ if is_thinking:
414
+ index, content_delta, stop_token = _read_until_stop(
415
+ index, text, [thinking_end_token, tool_calls_start_token]
416
+ )
417
+ reasoning_content = content_delta
418
+ assert stop_token == thinking_end_token, "Invalid thinking format"
419
+
420
+ index, content_delta, stop_token = _read_until_stop(
421
+ index, text, [eos_token, tool_calls_start_token]
422
+ )
423
+ summary_content = content_delta
424
+ if stop_token == tool_calls_start_token:
425
+ is_tool_calling = True
426
+ else:
427
+ assert stop_token == eos_token, "Invalid summary format"
428
+
429
+ if is_tool_calling:
430
+ index, stop_token, tool_calls = parse_tool_calls(index, text)
431
+
432
+ index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token])
433
+ assert not tool_ends_text, "Unexpected content after tool calls"
434
+
435
+ assert len(text) == index and stop_token in [eos_token, None], (
436
+ "Unexpected content at end"
437
+ )
438
+
439
+ for sp_token in [
440
+ bos_token,
441
+ eos_token,
442
+ thinking_start_token,
443
+ thinking_end_token,
444
+ dsml_token,
445
+ ]:
446
+ assert sp_token not in summary_content and sp_token not in reasoning_content, (
447
+ "Unexpected special token in content"
448
+ )
449
+
450
+ return {
451
+ "role": "assistant",
452
+ "content": summary_content,
453
+ "reasoning_content": reasoning_content,
454
+ "reasoning": reasoning_content,
455
+ "tool_calls": tool_calls_to_openai_format(tool_calls),
456
+ }
vllm/tokenizers/deepseekv32.py
new
+148
@@ -0,0 +1,148 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from pathlib import Path
5
+
6
+from transformers import BatchEncoding
7
+
8
+from .deepseek_v32_encoding import encode_messages
9
+from .hf import HfTokenizer, TokenizerLike
10
+from .registry import TokenizerRegistry
11
+
12
+
13
+@TokenizerRegistry.register("deepseek_v32")
14
+class DeepseekV32Tokenizer(HfTokenizer):
15
+ def __init__(self, tokenizer: TokenizerLike):
16
+ self.tokenizer = tokenizer
17
+ self.name_or_path = (
18
+ tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else ""
19
+ )
20
+
21
+ @classmethod
22
+ def from_pretrained(
23
+ cls,
24
+ path_or_repo_id: str | Path,
25
+ *args,
26
+ trust_remote_code: bool = False,
27
+ revision: str | None = None,
28
+ download_dir: str | None = None,
29
+ **kwargs,
30
+ ) -> "TokenizerLike":
31
+ tokenizer = super().from_pretrained(
32
+ path_or_repo_id,
33
+ *args,
34
+ trust_remote_code=trust_remote_code,
35
+ revision=revision,
36
+ download_dir=download_dir,
37
+ **kwargs,
38
+ )
39
+ return DeepseekV32Tokenizer(tokenizer)
40
+
41
+ def apply_chat_template(self, messages, tools=None, **kwargs):
42
+ thinking = kwargs.get("thinking", False)
43
+ thinking_mode = "thinking"
44
+ if not thinking:
45
+ thinking_mode = "chat"
46
+ messages = messages.copy()
47
+ drop_thinking = True
48
+ if tools is not None and len(tools) > 0:
49
+ messages.insert(0, {"role": "system"})
50
+ messages[0]["tools"] = tools
51
+ drop_thinking = False
52
+ encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
53
+ prompt_str = encode_messages(messages, **encode_config) # type: ignore
54
+ return prompt_str
55
+
56
+ @property
57
+ def all_special_tokens(self) -> list[str]:
58
+ return self.tokenizer.all_special_tokens
59
+
60
+ @property
61
+ def all_special_ids(self) -> list[int]:
62
+ return self.tokenizer.all_special_ids
63
+
64
+ @property
65
+ def bos_token_id(self) -> int:
66
+ return self.tokenizer.bos_token_id
67
+
68
+ @property
69
+ def eos_token_id(self) -> int:
70
+ return self.tokenizer.eos_token_id
71
+
72
+ @property
73
+ def pad_token_id(self) -> int:
74
+ return self.tokenizer.pad_token_id
75
+
76
+ @property
77
+ def is_fast(self) -> bool:
78
+ return self.tokenizer.is_fast
79
+
80
+ @property
81
+ def vocab_size(self) -> int:
82
+ return self.tokenizer.vocab_size
83
+
84
+ @property
85
+ def max_token_id(self) -> int:
86
+ return self.tokenizer.max_token_id
87
+
88
+ @property
89
+ def truncation_side(self) -> str:
90
+ return self.tokenizer.truncation_side
91
+
92
+ def __hash__(self) -> int:
93
+ return hash(id(self))
94
+
95
+ def __len__(self) -> int:
96
+ # </think> is an added token in DeepseekV32 tokenizer
97
+ return self.vocab_size + len(self.get_added_vocab())
98
+
99
+ def __call__(
100
+ self,
101
+ text: str | list[str],
102
+ text_pair: str | None = None,
103
+ add_special_tokens: bool = True,
104
+ truncation: bool = False,
105
+ max_length: int | None = None,
106
+ ) -> "BatchEncoding":
107
+ return self.tokenizer(
108
+ text,
109
+ text_pair=text_pair,
110
+ add_special_tokens=add_special_tokens,
111
+ truncation=truncation,
112
+ max_length=max_length,
113
+ )
114
+
115
+ def get_vocab(self) -> dict[str, int]:
116
+ return self.tokenizer.get_vocab()
117
+
118
+ def get_added_vocab(self) -> dict[str, int]:
119
+ return self.tokenizer.get_added_vocab()
120
+
121
+ def encode(
122
+ self,
123
+ text: str,
124
+ truncation: bool | None = None,
125
+ max_length: int | None = None,
126
+ add_special_tokens: bool = True,
127
+ ) -> list[int]:
128
+ return self.tokenizer.encode(
129
+ text,
130
+ truncation=truncation,
131
+ max_length=max_length,
132
+ add_special_tokens=add_special_tokens,
133
+ )
134
+
135
+ def convert_tokens_to_string(self, tokens: list[str]) -> str:
136
+ return self.tokenizer.convert_tokens_to_string(tokens)
137
+
138
+ def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
139
+ return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)
140
+
141
+ def convert_ids_to_tokens(
142
+ self,
143
+ ids: list[int],
144
+ skip_special_tokens: bool = False,
145
+ ) -> list[str]:
146
+ return self.tokenizer.convert_ids_to_tokens(
147
+ ids, skip_special_tokens=skip_special_tokens
148
+ )