A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Retry HF processing if "Already borrowed" error occurs (#25535)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Sep 24, 2025 at 13:32 UTC
fed8a9b107df3e27d57728c6911c7d308b871477
1 file changed
+20
vllm/inputs/registry.py
+20
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import time
4
from collections.abc import Mapping
5
from dataclasses import dataclass
6
from typing import TYPE_CHECKING, Any, Union
@@ -139,6 +140,9 @@ class InputProcessingContext(InputContext):
140
hf_processor: ProcessorMixin,
141
data: Mapping[str, object],
142
kwargs: Mapping[str, object] = {},
143
+ *,
144
+ num_tries: int = 1,
145
+ max_tries: int = 5,
146
) -> Union[BatchFeature, JSONTree]:
147
"""
148
Call `hf_processor` on the prompt `data`
@@ -180,6 +184,22 @@ class InputProcessingContext(InputContext):
184
return cast_output
185
186
except Exception as exc:
187
+ # See https://github.com/huggingface/tokenizers/issues/537
188
+ if (isinstance(exc, RuntimeError) and exc
189
+ and exc.args[0] == "Already borrowed"
190
+ and num_tries < max_tries):
191
+ logger.warning(
192
+ "Failed to acquire tokenizer in current thread. "
193
+ "Retrying (%d/%d)...", num_tries, max_tries)
194
+ time.sleep(0.5)
195
+ return self.call_hf_processor(
196
+ hf_processor,
197
+ data,
198
+ kwargs,
199
+ num_tries=num_tries + 1,
200
+ max_tries=max_tries,
201
+ )
202
+
203
msg = (f"Failed to apply {type(hf_processor).__name__} "
204
f"on data={data} with kwargs={allowed_kwargs}")
205