A high-throughput and memory-efficient inference and serving engine for LLMs
[Frontend] Remove AsyncMicrobatchTokenizer. (#45759)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jun 16, 2026 at 16:02 UTC
c4fd9794e9060531e98846706d5a4fdc573c2c19
2 files changed
+12
-219
vllm/renderers/base.py
+12
-16
@@ -38,10 +38,7 @@ from vllm.multimodal.processing import BaseMultiModalProcessor
38
from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs
39
from vllm.multimodal.registry import MultiModalTimingRegistry
40
from vllm.tokenizers import TokenizerLike
41
-from vllm.utils.async_utils import (
42
- AsyncMicrobatchTokenizer,
43
- make_async,
44
-)
41
+from vllm.utils.async_utils import make_async
42
from vllm.utils.counter import AtomicCounter
43
from vllm.utils.torch_utils import set_default_torch_num_threads
44
from vllm.v1.metrics.stats import MultiModalCacheStats
@@ -92,8 +89,9 @@ class BaseRenderer(ABC, Generic[_T]):
89
# to keep the asyncio event loop responsive under concurrent load.
90
self._mm_executor: Executor = self._executor
91
95
- # Lazy initialization since offline LLM doesn't use async
96
- self._async_tokenizer: AsyncMicrobatchTokenizer | None = None
92
+ # Offloading tokenizer encode & decode to thread pool.
93
+ self._async_tokenizer_encode = make_async(self._encode, executor=self._executor)
94
+ self._async_tokenizer_decode = make_async(self._decode, executor=self._executor)
95
96
self.mm_processor: BaseMultiModalProcessor | None = None
97
self._readonly_mm_processor: BaseMultiModalProcessor | None = None
@@ -146,13 +144,11 @@ class BaseRenderer(ABC, Generic[_T]):
144
145
return tokenizer
146
149
- def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:
150
- if self._async_tokenizer is None:
151
- self._async_tokenizer = AsyncMicrobatchTokenizer(
152
- self.get_tokenizer(), executor=self._executor
153
- )
147
+ def _decode(self, *args, **kwargs):
148
+ return self.get_tokenizer().decode(*args, **kwargs)
149
155
- return self._async_tokenizer
150
+ def _encode(self, *args, **kwargs):
151
+ return self.get_tokenizer().encode(*args, **kwargs)
152
153
def get_mm_processor(self) -> "BaseMultiModalProcessor":
154
if self.mm_processor is None:
@@ -436,8 +432,7 @@ class BaseRenderer(ABC, Generic[_T]):
432
prompt: TextPrompt,
433
params: TokenizeParams,
434
) -> TokensPrompt:
439
- tokenizer = self.get_async_tokenizer()
440
- prompt_token_ids = await tokenizer.encode(
435
+ prompt_token_ids = await self._async_tokenizer_encode(
436
prompt["prompt"],
437
**params.get_encode_kwargs(),
438
)
@@ -451,8 +446,9 @@ class BaseRenderer(ABC, Generic[_T]):
446
return prompt
447
448
async def _detokenize_prompt_async(self, prompt: TokensPrompt) -> TokensPrompt:
454
- tokenizer = self.get_async_tokenizer()
455
- prompt["prompt"] = await tokenizer.decode(prompt["prompt_token_ids"])
449
+ prompt["prompt"] = await self._async_tokenizer_decode(
450
+ prompt["prompt_token_ids"]
451
+ )
452
453
return prompt
454
vllm/utils/async_utils.py
-203
@@ -14,215 +14,12 @@ from concurrent.futures import Executor, ThreadPoolExecutor
14
from functools import partial
15
from typing import TYPE_CHECKING, TypeVar
16
17
-from transformers.tokenization_utils_base import BatchEncoding
17
from typing_extensions import ParamSpec
18
19
P = ParamSpec("P")
20
T = TypeVar("T")
21
22
24
-class AsyncMicrobatchTokenizer:
25
- """Asynchronous tokenizer with micro-batching.
26
-
27
- Pulls pending encode/decode requests from a queue and batches them
28
- up to reduce overhead. A single-thread ThreadPoolExecutor is used
29
- so the event loop stays responsive.
30
- """
31
-
32
- def __init__(
33
- self,
34
- tokenizer,
35
- max_batch_size: int = 32,
36
- batch_wait_timeout_s: float = 0.002,
37
- executor: ThreadPoolExecutor | None = None,
38
- ) -> None:
39
- self.tokenizer = tokenizer
40
- self.max_batch_size = max_batch_size
41
- self.batch_wait_timeout_s = batch_wait_timeout_s
42
-
43
- self._loop = asyncio.get_running_loop()
44
- self._queues: dict[
45
- tuple,
46
- asyncio.Queue[tuple[str, dict, Future] | tuple[list[int], Future]],
47
- ] = {}
48
- self._batcher_tasks: list[Task] = []
49
-
50
- # Single-thread executor for blocking tokenizer calls.
51
- # Accept an external executor to serialize with other tokenizer users.
52
- self._executor = executor or ThreadPoolExecutor(max_workers=1)
53
-
54
- # === Public async API ===
55
- async def __call__(self, prompt, **kwargs) -> BatchEncoding:
56
- result_future: Future = self._loop.create_future()
57
- key = self._queue_key("encode", kwargs)
58
- queue = self._get_queue(self._loop, key)
59
- await queue.put((prompt, kwargs, result_future))
60
- return await result_future
61
-
62
- async def encode(self, prompt, **kwargs) -> list[int]:
63
- return (await self(prompt, **kwargs)).input_ids
64
-
65
- async def decode(self, token_ids, **kwargs) -> str:
66
- result_future: Future = self._loop.create_future()
67
- key = self._queue_key("decode", kwargs)
68
- queue = self._get_queue(self._loop, key)
69
- await queue.put((token_ids, result_future))
70
- return await result_future
71
-
72
- # === Internal helpers ===
73
- def _get_queue(
74
- self, loop: asyncio.AbstractEventLoop, key: tuple
75
- ) -> asyncio.Queue[tuple[str, dict, Future] | tuple[list[int], Future]]:
76
- """Get the request queue for the given operation key, creating a new
77
- queue and batcher task if needed."""
78
- queue = self._queues.get(key)
79
- if queue is None:
80
- self._queues[key] = queue = asyncio.Queue()
81
- if key[0] == "encode":
82
- can_batch = key[1] != "other"
83
- coro = self._batch_encode_loop(queue, can_batch)
84
- else:
85
- assert key[0] == "decode", f"Unknown operation type: {key[0]}."
86
- coro = self._batch_decode_loop(queue)
87
- self._batcher_tasks.append(loop.create_task(coro))
88
- return queue
89
-
90
- async def _batch_encode_loop(self, queue: asyncio.Queue, can_batch: bool):
91
- """Batch incoming encode requests for efficiency."""
92
- while True:
93
- prompt, kwargs, result_future = await queue.get()
94
- prompts = [prompt]
95
- kwargs_list = [kwargs]
96
- result_futures = [result_future]
97
- deadline = self._loop.time() + self.batch_wait_timeout_s
98
-
99
- while len(prompts) < self.max_batch_size:
100
- timeout = deadline - self._loop.time()
101
- if timeout <= 0:
102
- break
103
- try:
104
- prompt, kwargs, result_future = await asyncio.wait_for(
105
- queue.get(), timeout
106
- )
107
- prompts.append(prompt)
108
- result_futures.append(result_future)
109
- if not can_batch:
110
- kwargs_list.append(kwargs)
111
- except asyncio.TimeoutError:
112
- break
113
-
114
- try:
115
- # If every request uses identical kwargs we can run a single
116
- # batched tokenizer call for a big speed-up.
117
- if can_batch and len(prompts) > 1:
118
- batch_encode_fn = partial(self.tokenizer, prompts, **kwargs)
119
- results = await self._loop.run_in_executor(
120
- self._executor, batch_encode_fn
121
- )
122
-
123
- for i, fut in enumerate(result_futures):
124
- if not fut.done():
125
- data = {k: v[i] for k, v in results.items()}
126
- fut.set_result(BatchEncoding(data))
127
- else:
128
- encode_fn = lambda prompts=prompts, kwargs=kwargs_list: [
129
- self.tokenizer(p, **kw) for p, kw in zip(prompts, kwargs)
130
- ]
131
- results = await self._loop.run_in_executor(
132
- self._executor, encode_fn
133
- )
134
-
135
- for fut, res in zip(result_futures, results):
136
- if not fut.done():
137
- fut.set_result(res)
138
- except Exception as e:
139
- for fut in result_futures:
140
- if not fut.done():
141
- fut.set_exception(e)
142
-
143
- async def _batch_decode_loop(self, queue: asyncio.Queue):
144
- """Batch incoming decode requests for efficiency."""
145
- while True:
146
- token_ids, result_future = await queue.get()
147
- token_ids_list = [token_ids]
148
- result_futures = [result_future]
149
- deadline = self._loop.time() + self.batch_wait_timeout_s
150
-
151
- while len(token_ids_list) < self.max_batch_size:
152
- timeout = deadline - self._loop.time()
153
- if timeout <= 0:
154
- break
155
- try:
156
- token_ids, result_future = await asyncio.wait_for(
157
- queue.get(), timeout
158
- )
159
- token_ids_list.append(token_ids)
160
- result_futures.append(result_future)
161
- except asyncio.TimeoutError:
162
- break
163
-
164
- try:
165
- # Perform a single batched decode call for all requests
166
- results = await self._loop.run_in_executor(
167
- self._executor, self.tokenizer.batch_decode, token_ids_list
168
- )
169
- for fut, res in zip(result_futures, results):
170
- if not fut.done():
171
- fut.set_result(res)
172
- except Exception as e:
173
- for fut in result_futures:
174
- if not fut.done():
175
- fut.set_exception(e)
176
-
177
- def _queue_key(self, op: str, kwargs: dict) -> tuple:
178
- """
179
- Return a normalized key describing operation + kwargs.
180
-
181
- - `add_special_tokens`: {True/False}
182
- - `truncation`: {True/False}
183
- - If `truncation` is False (`max_length` is None),
184
- returns a key for a can_batch queue.
185
- - If `truncation` is True and `max_length` is None or equals
186
- `tokenizer.model_max_length`, returns a key for a can_batch queue.
187
- - Otherwise, returns a key for a cannot_batch queue.
188
-
189
- Examples:
190
- - Decode: ("decode",)
191
- - Encode typical:
192
- ("encode", add_special_tokens, bool_truncation, max_length_label)
193
- - Fallback: ("encode", "other")
194
- """
195
-
196
- if op == "decode":
197
- return ("decode",)
198
-
199
- add_special_tokens = kwargs.get("add_special_tokens", True)
200
- truncation = kwargs.get("truncation", False)
201
- max_length = kwargs.get("max_length")
202
-
203
- if not truncation:
204
- return "encode", add_special_tokens, False, None
205
-
206
- model_max = getattr(self.tokenizer, "model_max_length", None)
207
- if max_length is None or (model_max is not None and max_length == model_max):
208
- return "encode", add_special_tokens, True, "model_max"
209
-
210
- return "encode", "other"
211
-
212
- def __del__(self):
213
- if (
214
- (tasks := getattr(self, "_batcher_tasks", None))
215
- and (loop := getattr(self, "_loop", None))
216
- and not loop.is_closed()
217
- ):
218
-
219
- def cancel_tasks():
220
- for task in tasks:
221
- task.cancel()
222
-
223
- loop.call_soon_threadsafe(cancel_tasks)
224
-
225
-
23
def cancel_task_threadsafe(task: Task):
24
if task and not task.done():
25
run_in_loop(task.get_loop(), task.cancel)