A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[BugFix] Fix tokenize asyncio task leak (#24677)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Sep 11, 2025 at 12:44 UTC
b971f91504b60f15909e1d6cc35e6468db4666e5
1 file changed
+30
-28
vllm/entrypoints/renderer.py
+30
-28
@@ -168,8 +168,8 @@ class BaseRenderer(ABC):
168
169
if isinstance(prompt_embeds, list):
170
return [_load_and_validate_embed(embed) for embed in prompt_embeds]
171
- else:
172
- return [_load_and_validate_embed(prompt_embeds)]
171
+
172
+ return [_load_and_validate_embed(prompt_embeds)]
173
174
175
class CompletionRenderer(BaseRenderer):
@@ -182,7 +182,7 @@ class CompletionRenderer(BaseRenderer):
182
AsyncMicrobatchTokenizer]] = None,
183
):
184
super().__init__(model_config, tokenizer)
185
- self.async_tokenizer_pool = async_tokenizer_pool or {}
185
+ self.async_tokenizer_pool = async_tokenizer_pool
186
self.async_tokenizer: Optional[AsyncMicrobatchTokenizer] = None
187
188
async def render_prompt(
@@ -208,23 +208,21 @@ class CompletionRenderer(BaseRenderer):
208
for prompt_input in batch_inputs:
209
if prompt_input["is_tokens"] is True:
210
# Token input
211
- detokenize_task = asyncio.create_task(
212
- # Note: detokenization is needed when echo is enabled,
213
- # where the input token IDs are decoded back to text.
214
- self._maybe_detokenize(prompt_input["content"],
215
- config.max_length,
216
- truncate_prompt_tokens,
217
- config.cache_salt,
218
- config.needs_detokenization))
219
- tasks.append(detokenize_task)
211
+ # Note: detokenization is needed when echo is enabled,
212
+ # where the input token IDs are decoded back to text.
213
+ task = self._maybe_detokenize(prompt_input["content"],
214
+ config.max_length,
215
+ truncate_prompt_tokens,
216
+ config.cache_salt,
217
+ config.needs_detokenization)
218
else:
219
# Text input
222
- tokenize_task = asyncio.create_task(
223
- self._tokenize(prompt_input["content"], config.max_length,
224
- truncate_prompt_tokens,
225
- config.add_special_tokens,
226
- config.cache_salt))
227
- tasks.append(tokenize_task)
220
+ task = self._tokenize(prompt_input["content"],
221
+ config.max_length,
222
+ truncate_prompt_tokens,
223
+ config.add_special_tokens,
224
+ config.cache_salt)
225
+ tasks.append(task)
226
227
# Wait for all text tokenization to finish
228
if tasks:
@@ -356,20 +354,24 @@ class CompletionRenderer(BaseRenderer):
354
355
def _get_async_tokenizer(self) -> AsyncMicrobatchTokenizer:
356
"""Get or create async tokenizer using shared pool."""
359
- if self.async_tokenizer is not None:
360
- return self.async_tokenizer
357
+ async_tokenizer = self.async_tokenizer
358
+ if async_tokenizer is not None:
359
+ return async_tokenizer
360
+
361
+ tokenizer = self.tokenizer
362
if self.tokenizer is None:
363
raise ValueError(
364
"No tokenizer available for text input processing")
365
365
- # Check shared pool first
366
- if self.tokenizer in self.async_tokenizer_pool:
367
- return self.async_tokenizer_pool[self.tokenizer]
368
-
369
- # Create new async tokenizer and add to pool
370
- self.async_tokenizer = AsyncMicrobatchTokenizer(self.tokenizer)
371
- self.async_tokenizer_pool[self.tokenizer] = self.async_tokenizer
372
- return self.async_tokenizer
366
+ if self.async_tokenizer_pool is None:
367
+ async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)
368
+ else:
369
+ async_tokenizer = self.async_tokenizer_pool.get(tokenizer)
370
+ if async_tokenizer is None:
371
+ async_tokenizer = AsyncMicrobatchTokenizer(tokenizer)
372
+ self.async_tokenizer_pool[tokenizer] = async_tokenizer
373
+ self.async_tokenizer = async_tokenizer
374
+ return async_tokenizer
375
376
def _create_tokens_prompt(
377
self,