A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 07:00 UTC
[Perf] Support stream interval for reducing host overhead (#27869)
Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Nick Hill <nhill@redhat.com>
elvischenv committed
Nov 14, 2025 at 02:21 UTC
5d6ce2b9601f3251487e44eb9e00c098101c4af6
6 files changed
+67
-5
tests/v1/engine/test_output_processor.py
+16
-2
@@ -49,10 +49,15 @@ def _ref_convert_id_to_token(
49
@pytest.mark.parametrize(
50
"request_output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]
51
)
52
+@pytest.mark.parametrize("stream_interval", [1, 5, 10])
53
def test_incremental_detokenization(
53
- request_output_kind: RequestOutputKind, dummy_test_vectors
54
+ request_output_kind: RequestOutputKind,
55
+ stream_interval: int,
56
+ dummy_test_vectors,
57
):
55
- output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=False)
58
+ output_processor = OutputProcessor(
59
+ dummy_test_vectors.tokenizer, log_stats=False, stream_interval=stream_interval
60
+ )
61
engine_core = MockEngineCore(tokens_list=dummy_test_vectors.generation_tokens)
62
63
# Make N requests.
@@ -104,9 +109,18 @@ def test_incremental_detokenization(
109
if request_id not in gen_strings:
110
gen_strings[request_id] = new_text
111
gen_tokens[request_id] = new_tokens
112
+ if request_output_kind == RequestOutputKind.DELTA:
113
+ assert len(new_tokens) == 1, f"{len(new_tokens)=}"
114
else:
115
gen_strings[request_id] += new_text
116
gen_tokens[request_id].extend(new_tokens)
117
+ if (
118
+ request_output_kind == RequestOutputKind.DELTA
119
+ and not request_output.finished
120
+ ):
121
+ assert len(new_tokens) >= stream_interval, (
122
+ f"{len(new_tokens)=}, {stream_interval=}"
123
+ )
124
125
# Confirmed tracked values matches what we expected.
126
for idx, (ref_gen_str, ref_gen_toks) in enumerate(
vllm/config/scheduler.py
+6
@@ -142,6 +142,12 @@ class SchedulerConfig:
142
speculative decoding and pipeline parallelism.
143
"""
144
145
+ stream_interval: int = Field(default=1, ge=1)
146
+ """The interval (or buffer size) for streaming in terms of token length.
147
+ A smaller value (1) makes streaming smoother by sending each token immediately,
148
+ while a larger value (e.g., 10) reduces host overhead and may increase throughput
149
+ by batching multiple tokens before sending."""
150
+
151
def get_scheduler_cls(self) -> type["SchedulerInterface"]:
152
if self.scheduler_cls is None:
153
if self.async_scheduling:
vllm/engine/arg_utils.py
+6
@@ -558,6 +558,8 @@ class EngineArgs:
558
559
async_scheduling: bool | None = SchedulerConfig.async_scheduling
560
561
+ stream_interval: int = SchedulerConfig.stream_interval
562
+
563
kv_sharing_fast_prefill: bool = CacheConfig.kv_sharing_fast_prefill
564
565
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
@@ -1067,6 +1069,9 @@ class EngineArgs:
1069
scheduler_group.add_argument(
1070
"--async-scheduling", **scheduler_kwargs["async_scheduling"]
1071
)
1072
+ scheduler_group.add_argument(
1073
+ "--stream-interval", **scheduler_kwargs["stream_interval"]
1074
+ )
1075
1076
# Compilation arguments
1077
compilation_kwargs = get_kwargs(CompilationConfig)
@@ -1562,6 +1567,7 @@ class EngineArgs:
1567
long_prefill_token_threshold=self.long_prefill_token_threshold,
1568
disable_hybrid_kv_cache_manager=self.disable_hybrid_kv_cache_manager,
1569
async_scheduling=self.async_scheduling,
1570
+ stream_interval=self.stream_interval,
1571
)
1572
1573
if not model_config.is_multimodal_model and self.default_mm_loras:
vllm/v1/engine/async_llm.py
+2
-1
@@ -120,8 +120,9 @@ class AsyncLLM(EngineClient):
120
)
121
122
# OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
123
+ stream_interval = self.vllm_config.scheduler_config.stream_interval
124
self.output_processor = OutputProcessor(
124
- self.tokenizer, log_stats=self.log_stats
125
+ self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
126
)
127
endpoint = self.observability_config.otlp_traces_endpoint
128
if endpoint is not None:
vllm/v1/engine/llm_engine.py
+2
-1
@@ -96,8 +96,9 @@ class LLMEngine:
96
)
97
98
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
99
+ stream_interval = self.vllm_config.scheduler_config.stream_interval
100
self.output_processor = OutputProcessor(
100
- self.tokenizer, log_stats=self.log_stats
101
+ self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval
102
)
103
endpoint = self.observability_config.otlp_traces_endpoint
104
if endpoint is not None:
vllm/v1/engine/output_processor.py
+35
-1
@@ -104,6 +104,7 @@ class RequestState:
104
arrival_time: float,
105
queue: RequestOutputCollector | None,
106
log_stats: bool,
107
+ stream_interval: int,
108
top_p: float | None = None,
109
n: int | None = None,
110
temperature: float | None = None,
@@ -131,6 +132,10 @@ class RequestState:
132
133
self.stats = RequestStateStats(arrival_time=arrival_time) if log_stats else None
134
135
+ # Stream Interval
136
+ self.stream_interval = stream_interval
137
+ self.sent_tokens_offset = 0 # Offset of sent tokens
138
+
139
@classmethod
140
def from_new_request(
141
cls,
@@ -141,6 +146,7 @@ class RequestState:
146
request_index: int,
147
queue: RequestOutputCollector | None,
148
log_stats: bool,
149
+ stream_interval: int,
150
) -> "RequestState":
151
if sampling_params := request.sampling_params:
152
if not sampling_params.detokenize:
@@ -188,6 +194,7 @@ class RequestState:
194
arrival_time=request.arrival_time,
195
queue=queue,
196
log_stats=log_stats,
197
+ stream_interval=stream_interval,
198
)
199
200
def make_request_output(
@@ -205,6 +212,29 @@ class RequestState:
212
# Only the final output is required in FINAL_ONLY mode.
213
return None
214
215
+ if self.stream_interval > 1:
216
+ assert self.detokenizer is not None
217
+
218
+ # Send output request only when
219
+ # 1. It has finished, or
220
+ # 2. It is the first token, or
221
+ # 3. It has reached the stream interval number of tokens
222
+ if not (
223
+ finished
224
+ or self.sent_tokens_offset == 0
225
+ or len(self.detokenizer.output_token_ids) - self.sent_tokens_offset
226
+ >= self.stream_interval
227
+ ):
228
+ return None
229
+
230
+ if self.output_kind == RequestOutputKind.DELTA:
231
+ # Send tokens from the offset in DELTA mode, otherwise all
232
+ # tokens are sent.
233
+ new_token_ids = self.detokenizer.output_token_ids[
234
+ self.sent_tokens_offset :
235
+ ]
236
+ self.sent_tokens_offset = len(self.detokenizer.output_token_ids)
237
+
238
request_id = self.request_id
239
if pooling_output is not None:
240
return self._new_request_output(
@@ -310,9 +340,12 @@ class RequestState:
340
class OutputProcessor:
341
"""Process EngineCoreOutputs into RequestOutputs."""
342
313
- def __init__(self, tokenizer: AnyTokenizer, log_stats: bool):
343
+ def __init__(
344
+ self, tokenizer: AnyTokenizer, log_stats: bool, stream_interval: int = 1
345
+ ):
346
self.log_stats = log_stats
347
self.tokenizer = tokenizer
348
+ self.stream_interval = stream_interval
349
self.request_states: dict[str, RequestState] = {}
350
self.parent_requests: dict[str, ParentRequest] = {}
351
self.lora_states = LoRARequestStates(log_stats)
@@ -385,6 +418,7 @@ class OutputProcessor:
418
request_index=request_index,
419
queue=queue,
420
log_stats=self.log_stats,
421
+ stream_interval=self.stream_interval,
422
)
423
self.request_states[request_id] = req_state
424
if parent_req: