A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
[Bugfix] Fix `RuntimeError: Already borrowed` by adding thread-safe Hugging Face fast-tokenizer wrappers (#41181)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
yzong-rh committed
May 5, 2026 at 10:04 UTC
20dcd984f9a49b8dc69c400486eed50953cb16cf
5 files changed
+107
-35
vllm/multimodal/processing/context.py
-22
@@ -268,28 +268,6 @@ class InputProcessingContext:
268
try:
269
output = hf_processor(**data, **allowed_kwargs)
270
except Exception as exc:
271
- # See https://github.com/huggingface/tokenizers/issues/537
272
- if (
273
- isinstance(exc, RuntimeError)
274
- and exc
275
- and exc.args[0] == "Already borrowed"
276
- and num_tries < max_tries
277
- ):
278
- logger.warning(
279
- "Failed to acquire tokenizer in current thread. "
280
- "Retrying (%d/%d)...",
281
- num_tries,
282
- max_tries,
283
- )
284
- time.sleep(0.5)
285
- return self.call_hf_processor(
286
- hf_processor,
287
- data,
288
- kwargs,
289
- num_tries=num_tries + 1,
290
- max_tries=max_tries,
291
- )
292
-
271
msg = (
272
f"Failed to apply {type(hf_processor).__name__} "
273
f"on data={data} with kwargs={allowed_kwargs}"
vllm/renderers/base.py
+2
-11
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
-import copy
4
import time
5
from abc import ABC, abstractmethod
6
from collections.abc import Mapping, Sequence
@@ -108,17 +107,10 @@ class BaseRenderer(ABC, Generic[_T]):
107
if mm_registry.supports_multimodal_inputs(config.model_config):
108
mm_processor_cache = mm_registry.processor_cache_from_config(config)
109
111
- # Deep-copy the tokenizer so the multimodal processor gets its
112
- # own Rust tokenizer backend. Without this, concurrent access
113
- # from AsyncMicrobatchTokenizer and call_hf_processor causes
114
- # "RuntimeError: Already borrowed" from the Rust RefCell.
115
- # See: https://github.com/huggingface/tokenizers/issues/537
116
- mm_tokenizer = copy.deepcopy(tokenizer)
117
-
110
with set_default_torch_num_threads():
111
self.mm_processor = mm_registry.create_processor(
112
config.model_config,
121
- tokenizer=mm_tokenizer,
113
+ tokenizer=self.tokenizer,
114
cache=mm_processor_cache,
115
)
116
@@ -130,11 +122,10 @@ class BaseRenderer(ABC, Generic[_T]):
122
# requests don't pollute the sender cache.
123
ro_cache = mm_registry.processor_only_cache_from_config(config)
124
if ro_cache is not None:
133
- ro_tokenizer = copy.deepcopy(tokenizer)
125
with set_default_torch_num_threads():
126
self._readonly_mm_processor = mm_registry.create_processor(
127
config.model_config,
137
- tokenizer=ro_tokenizer,
128
+ tokenizer=self.tokenizer,
129
cache=ro_cache,
130
)
131
vllm/renderers/hf.py
+15
-1
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from __future__ import annotations
4
5
+import copy
6
import inspect
7
import itertools
8
import weakref
@@ -42,7 +43,7 @@ from vllm.multimodal.processing.processor import (
43
apply_token_matches,
44
find_mm_placeholders,
45
)
45
-from vllm.tokenizers.hf import HfTokenizer
46
+from vllm.tokenizers.hf import HfTokenizer, maybe_make_thread_pool
47
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
48
from vllm.transformers_utils.processor import cached_get_processor
49
from vllm.utils.async_utils import make_async
@@ -785,6 +786,14 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
786
config: VllmConfig,
787
tokenizer: HfTokenizer | None,
788
) -> None:
789
+ # Ensure the og tokenizer is never modified by maybe_make_thread_pool
790
+ tokenizer = copy.copy(tokenizer)
791
+ if (
792
+ # Skip for mock configs and tokenizers
793
+ getattr(config.model_config, "enable_prompt_embeds", False)
794
+ and isinstance(tokenizer, HfTokenizer)
795
+ ):
796
+ _ensure_prompt_embeds_placeholder_token(tokenizer)
797
super().__init__(config, tokenizer)
798
799
self.use_unified_vision_chunk = getattr(
@@ -795,6 +804,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
804
safe_apply_chat_template, executor=self._executor
805
)
806
807
+ if self.tokenizer is not None:
808
+ maybe_make_thread_pool(
809
+ self.tokenizer, config.model_config.renderer_num_workers + 1
810
+ )
811
+
812
def render_messages(
813
self,
814
messages: list[ChatCompletionMessageParam],
vllm/tokenizers/__init__.py
+2
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from .hf import maybe_make_thread_pool
5
from .protocol import TokenizerLike
6
from .registry import (
7
TokenizerRegistry,
@@ -15,4 +16,5 @@ __all__ = [
16
"cached_get_tokenizer",
17
"get_tokenizer",
18
"cached_tokenizer_from_config",
19
+ "maybe_make_thread_pool",
20
]
vllm/tokenizers/hf.py
+88
-1
@@ -2,8 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import contextlib
4
import copy
5
+import queue
6
from pathlib import Path
6
-from typing import TypeAlias
7
+from typing import TypeAlias, TypeVar
8
9
from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
10
@@ -12,6 +13,92 @@ from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_co
13
from .protocol import TokenizerLike
14
15
HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast
16
+_T = TypeVar("_T", bound=TokenizerLike)
17
+
18
+
19
+class ThreadSafeHFTokenizerMixin:
20
+ """Mixin class for thread-safe HF fast tokenizers."""
21
+
22
+ pass
23
+
24
+
25
+def maybe_make_thread_pool(tokenizer: _T, copies: int = 1):
26
+ """
27
+ If `tokenizer` is a `PreTrainedTokenizerFast`, modify the tokenizer
28
+ in-place to make the public interface thread-safe by routing calls
29
+ through a deep-copied tokenizer pool.
30
+
31
+ Note that:
32
+ - Only ``TokenizerLike``'s public interface is thread-safe.
33
+ This doesn't include ``_tokenizer`` property nor any mutation
34
+ methods like ``add_special_tokens`` or ``add_tokens``.
35
+ - Adjacent method calls could happen on different deep copies.
36
+ """
37
+ if not isinstance(tokenizer, PreTrainedTokenizerFast) or isinstance(
38
+ tokenizer, ThreadSafeHFTokenizerMixin
39
+ ):
40
+ return tokenizer
41
+
42
+ og_tokenizer = copy.copy(tokenizer)
43
+
44
+ tokenizer_pool: queue.Queue[PreTrainedTokenizerFast] = queue.Queue()
45
+ for _ in range(copies):
46
+ tokenizer_pool.put(copy.deepcopy(og_tokenizer))
47
+
48
+ @contextlib.contextmanager
49
+ def _borrow_from_pool():
50
+ try:
51
+ tok = tokenizer_pool.get_nowait()
52
+ yield tok
53
+ except queue.Empty:
54
+ tok = copy.deepcopy(og_tokenizer)
55
+ yield tok
56
+ finally:
57
+ tokenizer_pool.put(tok)
58
+
59
+ class TokenizerPool(tokenizer.__class__, ThreadSafeHFTokenizerMixin): # type: ignore
60
+ def apply_chat_template(self, *args, **kwargs):
61
+ with _borrow_from_pool() as tok:
62
+ return tok.apply_chat_template(*args, **kwargs)
63
+
64
+ def batch_decode(self, *args, **kwargs):
65
+ with _borrow_from_pool() as tok:
66
+ return tok.batch_decode(*args, **kwargs)
67
+
68
+ def batch_encode(self, *args, **kwargs):
69
+ with _borrow_from_pool() as tok:
70
+ return tok.batch_encode(*args, **kwargs)
71
+
72
+ def convert_tokens_to_ids(self, *args, **kwargs):
73
+ with _borrow_from_pool() as tok:
74
+ return tok.convert_tokens_to_ids(*args, **kwargs)
75
+
76
+ def convert_ids_to_tokens(self, *args, **kwargs):
77
+ with _borrow_from_pool() as tok:
78
+ return tok.convert_ids_to_tokens(*args, **kwargs)
79
+
80
+ def convert_tokens_to_string(self, *args, **kwargs):
81
+ with _borrow_from_pool() as tok:
82
+ return tok.convert_tokens_to_string(*args, **kwargs)
83
+
84
+ def decode(self, *args, **kwargs):
85
+ with _borrow_from_pool() as tok:
86
+ return tok.decode(*args, **kwargs)
87
+
88
+ def encode(self, *args, **kwargs):
89
+ with _borrow_from_pool() as tok:
90
+ return tok.encode(*args, **kwargs)
91
+
92
+ def __call__(self, *args, **kwargs):
93
+ with _borrow_from_pool() as tok:
94
+ return tok(*args, **kwargs)
95
+
96
+ def __reduce__(self):
97
+ return maybe_make_thread_pool, (og_tokenizer, copies)
98
+
99
+ TokenizerPool.__name__ = f"TokenizerPool{og_tokenizer.__class__.__name__}"
100
+
101
+ tokenizer.__class__ = TokenizerPool
102
103
104
def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: