[Perf][Rust Frontend] Coalesce decoded chunks per engine update (#55012)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Sep 3, 2026 at 08:44 UTC
ad127d9a0fb16c92de563e674e3737463e7f8688
1 file changed
+7
-4
rust/src/text/src/output/decoded.rs
+7
-4
@@ -201,11 +201,14 @@ pub async fn decoded_text_event_stream(
201
202
break;
203
}
204
+ }
205
205
- // TODO: avoid generating a chunk every time a token is pushed
206
- if intermediate && let Some(chunk) = decoder.next_chunk() {
207
- decoded.append(chunk);
208
- }
206
+ // Coalesce output per nonterminal engine update; terminal updates flush below.
207
+ if intermediate
208
+ && finish_reason.is_none()
209
+ && let Some(chunk) = decoder.next_chunk()
210
+ {
211
+ decoded.append(chunk);
212
}
213
214
let mut new_token_ids = output.token_ids;