[Perf][Frontend] Offload streaming derender detokenization (#57528)
Signed-off-by: Shrey Gajjar <shreygajjar007@gmail.com>
SHREY GAJJAR committed
Sep 24, 2026 at 03:39 UTC
5747d4500a085496b874bd03c5cafb01f51ace56
2 files changed
+56
-11
tests/entrypoints/scale_out/derender/test_derender_stream.py
+50
-7
@@ -18,6 +18,8 @@ Tests are split into two layers:
18
"""
19
20
import json
21
+import threading
22
+from concurrent.futures import ThreadPoolExecutor
23
24
import pytest
25
import pytest_asyncio
@@ -214,7 +216,7 @@ def tokenizer():
216
217
218
@pytest.fixture(scope="module")
217
-def derenderer(tokenizer):
219
+def derenderer(tokenizer, request):
220
"""Construct a minimal OnlineDerenderer backed by a stub renderer."""
221
from unittest.mock import MagicMock
222
@@ -223,6 +225,10 @@ def derenderer(tokenizer):
225
renderer = MagicMock()
226
renderer.get_tokenizer.return_value = tokenizer
227
228
+ executor = ThreadPoolExecutor(max_workers=1)
229
+ request.addfinalizer(executor.shutdown)
230
+ renderer._executor = executor
231
+
232
model_config = MagicMock()
233
model_config.hf_config.model_type = "llama"
234
model_config.model = MODEL_NAME
@@ -244,13 +250,11 @@ def derenderer(tokenizer):
250
def parsed_derenderer(tokenizer):
251
"""OnlineDerenderer with `_FakeParser` wired in as `self.parser`.
252
247
- The parser configured path runs `_derender_chat_stream_parsed` via
248
- `make_async(..., executor=renderer._executor)`, so unlike
249
- `derenderer` above (whose streaming paths never touch the executor),
250
- `renderer._executor` must be a real `ThreadPoolExecutor` here.
253
+ The parser-configured path runs `_derender_chat_stream_parsed` via
254
+ `make_async(..., executor=renderer._executor)`, so
255
+ `renderer._executor` must be a real `ThreadPoolExecutor`.
256
`loop.run_in_executor` cannot submit work to a `MagicMock`.
257
"""
253
- from concurrent.futures import ThreadPoolExecutor
258
from unittest.mock import MagicMock
259
260
from vllm.renderers.online_derenderer import OnlineDerenderer
@@ -390,6 +394,46 @@ class TestDetokenizeDelta:
394
assert results[0] == self._one_shot(tokenizer, token_ids)
395
396
397
+@pytest.mark.asyncio
398
+@pytest.mark.parametrize(
399
+ "stream_method",
400
+ ["derender_chat_stream", "derender_completion_stream"],
401
+)
402
+async def test_streaming_detokenization_runs_off_event_loop(
403
+ derenderer, monkeypatch, stream_method
404
+):
405
+ """Streaming detokenization runs on the renderer executor."""
406
+ import vllm.renderers.online_derenderer as online_derenderer_module
407
+
408
+ event_loop_thread_id = threading.get_ident()
409
+ executor_thread_id = derenderer.renderer._executor.submit(
410
+ threading.get_ident
411
+ ).result()
412
+
413
+ detokenize_thread_id = None
414
+ original_detokenize = online_derenderer_module.detokenize_incrementally
415
+
416
+ def record_detokenize_thread(*args, **kwargs):
417
+ nonlocal detokenize_thread_id
418
+ detokenize_thread_id = threading.get_ident()
419
+ return original_detokenize(*args, **kwargs)
420
+
421
+ monkeypatch.setattr(
422
+ online_derenderer_module,
423
+ "detokenize_incrementally",
424
+ record_detokenize_thread,
425
+ )
426
+
427
+ await getattr(derenderer, stream_method)(
428
+ model=MODEL_NAME,
429
+ generate_chunk=_make_stream_chunk([10]),
430
+ )
431
+
432
+ assert detokenize_thread_id is not None
433
+ assert detokenize_thread_id == executor_thread_id
434
+ assert detokenize_thread_id != event_loop_thread_id
435
+
436
+
437
class TestDerenderCompletionStream:
438
"""derender_completion_stream: streaming output parity with one shot."""
439
@@ -875,7 +919,6 @@ def harmony_derenderer(harmony_tokenizer):
919
with a real tokenizer and parser, so the replay path is exercised
920
against Harmony's actual channel grammar rather than a stub.
921
"""
878
- from concurrent.futures import ThreadPoolExecutor
922
from unittest.mock import MagicMock
923
924
from vllm.parser.harmony import HarmonyParser
vllm/renderers/online_derenderer.py
+6
-4
@@ -101,8 +101,10 @@ class OnlineDerenderer:
101
self._derender_completion_async = make_async(
102
self._derender_completion, executor=renderer._executor
103
)
104
- # Replay is O(n) per chunk (unlike the O(delta) detok-only paths),
105
- # so it must not run on the event loop either.
104
+ self._detokenize_delta_async = make_async(
105
+ self._detokenize_delta, executor=renderer._executor
106
+ )
107
+ # Replay is O(n) per chunk, so it must not run on the event loop.
108
self._derender_chat_stream_parsed_async = make_async(
109
self._derender_chat_stream_parsed, executor=renderer._executor
110
)
@@ -362,7 +364,7 @@ class OnlineDerenderer:
364
365
for choice in generate_chunk.choices:
366
delta_tids = choice.token_ids or []
365
- new_text, updated_state = self._detokenize_delta(
367
+ new_text, updated_state = await self._detokenize_delta_async(
368
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
369
)
370
@@ -716,7 +718,7 @@ class OnlineDerenderer:
718
719
for choice in generate_chunk.choices:
720
delta_tids = choice.token_ids or []
719
- new_text, updated_state = self._detokenize_delta(
721
+ new_text, updated_state = await self._detokenize_delta_async(
722
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
723
)
724
stream_choices.append(