[Renderer] Offload derender CPU work to renderer thread pool (#49396)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Guan-Ming Chiu committed
Jul 22, 2026 at 14:55 UTC
9df2f91232bfb2bbd66e3106af74586403b7fde3
1 file changed
+24
vllm/renderers/online_derenderer.py
+24
@@ -24,6 +24,7 @@ from vllm.parser import Parser, ParserManager
24
from vllm.renderers import BaseRenderer
25
from vllm.tokenizers import TokenizerLike
26
from vllm.utils import random_uuid
27
+from vllm.utils.async_utils import make_async
28
29
logger = init_logger(__name__)
30
@@ -73,10 +74,26 @@ class OnlineDerenderer:
74
self.supports_browsing = False
75
self.supports_code_interpreter = False
76
77
+ # Detokenization, logprob resolution and parsing are CPU-bound;
78
+ # offload them in one hop to keep the event loop responsive.
79
+ self._derender_chat_async = make_async(
80
+ self._derender_chat, executor=renderer._executor
81
+ )
82
+ self._derender_completion_async = make_async(
83
+ self._derender_completion, executor=renderer._executor
84
+ )
85
+
86
async def derender_chat(
87
self,
88
generate_response: GenerateResponse,
89
chat_request: ChatCompletionRequest | None = None,
90
+ ) -> list[ChatCompletionResponseChoice]:
91
+ return await self._derender_chat_async(generate_response, chat_request)
92
+
93
+ def _derender_chat(
94
+ self,
95
+ generate_response: GenerateResponse,
96
+ chat_request: ChatCompletionRequest | None = None,
97
) -> list[ChatCompletionResponseChoice]:
98
tokenizer = self.renderer.get_tokenizer()
99
choices: list[ChatCompletionResponseChoice] = []
@@ -172,6 +189,13 @@ class OnlineDerenderer:
189
self,
190
generate_responses: list[GenerateResponse],
191
prompt_tokens: list[int] | None = None,
192
+ ) -> tuple[list[CompletionResponseChoice], int, int]:
193
+ return await self._derender_completion_async(generate_responses, prompt_tokens)
194
+
195
+ def _derender_completion(
196
+ self,
197
+ generate_responses: list[GenerateResponse],
198
+ prompt_tokens: list[int] | None = None,
199
) -> tuple[list[CompletionResponseChoice], int, int]:
200
n = len(generate_responses)
201
prompt_tokens_list: list[int] = (