A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
[Bugfix] Fix `RuntimeError: Already borrowed` that degrades VLM serving throughput under concurrent load. (#36557)
Signed-off-by: hallerite <hallerite@users.noreply.github.com> Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
hallerite committed
Mar 9, 2026 at 22:30 UTC
d0cd736caadafea1ec1721737af432d8b0a7e919
1 file changed
+9
-1
vllm/renderers/base.py
+9
-1
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import asyncio
4
+import copy
5
import time
6
from abc import ABC, abstractmethod
7
from collections.abc import Mapping, Sequence
@@ -90,10 +91,17 @@ class BaseRenderer(ABC, Generic[_T]):
91
92
mm_processor_cache = mm_registry.processor_cache_from_config(config)
93
94
+ # Deep-copy the tokenizer so the multimodal processor gets its
95
+ # own Rust tokenizer backend. Without this, concurrent access
96
+ # from AsyncMicrobatchTokenizer and call_hf_processor causes
97
+ # "RuntimeError: Already borrowed" from the Rust RefCell.
98
+ # See: https://github.com/huggingface/tokenizers/issues/537
99
+ mm_tokenizer = copy.deepcopy(tokenizer)
100
+
101
with set_default_torch_num_threads():
102
self.mm_processor = mm_registry.create_processor(
103
config.model_config,
96
- tokenizer=tokenizer,
104
+ tokenizer=mm_tokenizer,
105
cache=mm_processor_cache,
106
)
107