A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
Add output streaming support to multi-step + async while ensuring RequestOutput obj reuse (#8335)
Alexander Matveev committed
Sep 23, 2024 at 18:38 UTC
1a2aef3e59f5429299618bd3b242833cb377f554
7 files changed
+142
-42
tests/entrypoints/openai/test_accuracy.py
+5
-1
@@ -19,7 +19,11 @@ FILTER = "exact_match,strict-match"
19
RTOL = 0.03
20
EXPECTED_VALUE = 0.58
21
DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
22
-MORE_ARGS_LIST = [["--enable-chunked-prefill"], ["--num-scheduler-steps", "8"]]
22
+MORE_ARGS_LIST = [
23
+ ["--enable-chunked-prefill"], # Chunked
24
+ ["--num-scheduler-steps", "8"], # MS
25
+ ["--num-scheduler-steps", "8", "--multi-step-stream-outputs"] # MS+Stream
26
+]
27
28
29
@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
vllm/config.py
+2
@@ -960,6 +960,7 @@ class SchedulerConfig:
960
is_multimodal_model: bool = False,
961
preemption_mode: Optional[str] = None,
962
num_scheduler_steps: int = 1,
963
+ multi_step_stream_outputs: bool = False,
964
send_delta_data: bool = False) -> None:
965
if max_num_batched_tokens is None:
966
if enable_chunked_prefill:
@@ -1000,6 +1001,7 @@ class SchedulerConfig:
1001
self.embedding_mode = embedding_mode
1002
self.preemption_mode = preemption_mode
1003
self.num_scheduler_steps = num_scheduler_steps
1004
+ self.multi_step_stream_outputs = multi_step_stream_outputs
1005
self.send_delta_data = send_delta_data
1006
self._verify_args()
1007
vllm/engine/arg_utils.py
+6
@@ -145,6 +145,7 @@ class EngineArgs:
145
max_cpu_loras: Optional[int] = None
146
device: str = 'auto'
147
num_scheduler_steps: int = 1
148
+ multi_step_stream_outputs: bool = False
149
ray_workers_use_nsight: bool = False
150
num_gpu_blocks_override: Optional[int] = None
151
num_lookahead_slots: int = 0
@@ -595,6 +596,10 @@ class EngineArgs:
596
help=('Maximum number of forward steps per '
597
'scheduler call.'))
598
599
+ parser.add_argument(
600
+ '--multi-step-stream-outputs',
601
+ action='store_true',
602
+ help='If True, then multi-step will stream outputs for every step')
603
parser.add_argument(
604
'--scheduler-delay-factor',
605
type=float,
@@ -999,6 +1004,7 @@ class EngineArgs:
1004
is_multimodal_model=model_config.is_multimodal_model,
1005
preemption_mode=self.preemption_mode,
1006
num_scheduler_steps=self.num_scheduler_steps,
1007
+ multi_step_stream_outputs=self.multi_step_stream_outputs,
1008
send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1009
and parallel_config.use_ray),
1010
)
vllm/engine/llm_engine.py
+28
-9
@@ -95,7 +95,7 @@ class OutputData(NamedTuple):
95
96
class SchedulerContext:
97
98
- def __init__(self):
98
+ def __init__(self, multi_step_stream_outputs: bool = False):
99
self.output_queue: Deque[OutputData] = deque()
100
self.request_outputs: List[Union[RequestOutput,
101
EmbeddingRequestOutput]] = []
@@ -103,6 +103,8 @@ class SchedulerContext:
103
List[SequenceGroupMetadata]] = None
104
self.scheduler_outputs: Optional[SchedulerOutputs] = None
105
106
+ self.multi_step_stream_outputs: bool = multi_step_stream_outputs
107
+
108
def append_output(self, outputs: List[SamplerOutput],
109
seq_group_metadata_list: List[SequenceGroupMetadata],
110
scheduler_outputs: SchedulerOutputs, is_async: bool,
@@ -219,6 +221,7 @@ class LLMEngine:
221
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
222
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
223
input_registry: InputRegistry = INPUT_REGISTRY,
224
+ use_cached_outputs: bool = False,
225
) -> None:
226
logger.info(
227
"Initializing an LLM engine (v%s) with config: "
@@ -234,8 +237,9 @@ class LLMEngine:
237
"quantization_param_path=%s, device_config=%s, "
238
"decoding_config=%r, observability_config=%r, "
239
"seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
237
- "num_scheduler_steps=%d, enable_prefix_caching=%s, "
238
- "use_async_output_proc=%s, mm_processor_kwargs=%s)",
240
+ "num_scheduler_steps=%d, multi_step_stream_outputs=%s, "
241
+ "enable_prefix_caching=%s, use_async_output_proc=%s, "
242
+ "use_cached_outputs=%s, mm_processor_kwargs=%s)",
243
VLLM_VERSION,
244
model_config.model,
245
speculative_config,
@@ -266,8 +270,10 @@ class LLMEngine:
270
model_config.served_model_name,
271
scheduler_config.use_v2_block_manager,
272
scheduler_config.num_scheduler_steps,
273
+ scheduler_config.multi_step_stream_outputs,
274
cache_config.enable_prefix_caching,
275
model_config.use_async_output_proc,
276
+ use_cached_outputs,
277
model_config.mm_processor_kwargs,
278
)
279
# TODO(woosuk): Print more configs in debug mode.
@@ -287,6 +293,7 @@ class LLMEngine:
293
self.observability_config = observability_config or ObservabilityConfig(
294
)
295
self.log_stats = log_stats
296
+ self.use_cached_outputs = use_cached_outputs
297
298
if not self.model_config.skip_tokenizer_init:
299
self.tokenizer = self._init_tokenizer()
@@ -379,7 +386,8 @@ class LLMEngine:
386
]
387
388
self.scheduler_contexts = [
382
- SchedulerContext()
389
+ SchedulerContext(multi_step_stream_outputs=self.scheduler_config.
390
+ multi_step_stream_outputs)
391
for _ in range(self.parallel_config.pipeline_parallel_size)
392
]
393
@@ -998,7 +1006,8 @@ class LLMEngine:
1006
1007
seq_group = scheduled_seq_group.seq_group
1008
seq_group.maybe_set_first_token_time(now)
1001
- request_output = RequestOutputFactory.create(seq_group)
1009
+ request_output = RequestOutputFactory.create(
1010
+ seq_group, use_cache=self.use_cached_outputs)
1011
if request_output:
1012
ctx.request_outputs.append(request_output)
1013
@@ -1019,8 +1028,8 @@ class LLMEngine:
1028
for scheduler in self.scheduler:
1029
scheduler.free_finished_seq_groups()
1030
1022
- # For multi-step, do not create outputs each iteration
1023
- if not is_last_step:
1031
+ # For multi-step without streaming, don't create outputs each iteration
1032
+ if not is_last_step and not ctx.multi_step_stream_outputs:
1033
# Immediately process request outputs here (if callback is given)
1034
if (finished_now
1035
and self.process_request_outputs_callback is not None):
@@ -1037,17 +1046,27 @@ class LLMEngine:
1046
1047
seq_group = scheduled_seq_group.seq_group
1048
seq_group.maybe_set_first_token_time(now)
1040
- request_output = RequestOutputFactory.create(seq_group)
1049
+ request_output = RequestOutputFactory.create(
1050
+ seq_group, use_cache=self.use_cached_outputs)
1051
if request_output:
1052
ctx.request_outputs.append(request_output)
1053
1054
+ # For multi-step with streaming, create outputs each iteration
1055
+ if not is_last_step and ctx.multi_step_stream_outputs:
1056
+ # Immediately process request outputs here (if callback is given)
1057
+ if self.process_request_outputs_callback is not None:
1058
+ self.process_request_outputs_callback(ctx.request_outputs)
1059
+ ctx.request_outputs.clear()
1060
+ return
1061
+
1062
for seq_group in scheduler_outputs.ignored_seq_groups:
1063
params = seq_group.sampling_params
1064
if params is not None and params.output_kind == (
1065
RequestOutputKind.DELTA) and not seq_group.is_finished():
1066
continue
1067
1050
- request_output = RequestOutputFactory.create(seq_group)
1068
+ request_output = RequestOutputFactory.create(
1069
+ seq_group, use_cache=self.use_cached_outputs)
1070
if request_output:
1071
ctx.request_outputs.append(request_output)
1072
vllm/engine/multiprocessing/engine.py
+8
-1
@@ -66,7 +66,14 @@ class MQLLMEngine:
66
*args,
67
log_requests: bool = True,
68
**kwargs) -> None:
69
- self.engine = LLMEngine(*args, **kwargs)
69
+ # For MQLLMEngine, we can use cached outputs, since each new request
70
+ # output is immediately pickled and send over the socket, which frees
71
+ # the python object to be reused again.
72
+ use_cached_outputs = True
73
+
74
+ self.engine = LLMEngine(*args,
75
+ **kwargs,
76
+ use_cached_outputs=use_cached_outputs)
77
self.log_requests = log_requests
78
79
self.use_async_sockets = use_async_sockets
vllm/outputs.py
+74
-22
@@ -114,17 +114,28 @@ class RequestOutput:
114
self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116
@classmethod
117
- def from_seq_group(cls,
118
- seq_group: SequenceGroup) -> Optional["RequestOutput"]:
117
+ def from_seq_group(cls, seq_group: SequenceGroup,
118
+ use_cache: bool) -> Optional["RequestOutput"]:
119
sampling_params = seq_group.sampling_params
120
if sampling_params is None:
121
raise ValueError(
122
"Sampling parameters are missing for a CompletionRequest.")
123
+
124
finished = seq_group.is_finished()
125
if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
126
not finished):
127
return None
128
129
+ # Init cache (if needed)
130
+ if use_cache and seq_group.cached_request_output is None:
131
+ seq_group.cached_request_output = RequestOutput( # type: ignore
132
+ request_id="",
133
+ prompt=None,
134
+ prompt_token_ids=[],
135
+ prompt_logprobs=None,
136
+ outputs=[],
137
+ finished=False)
138
+
139
seqs = seq_group.get_seqs()
140
if len(seqs) == 1:
141
top_n_seqs = seqs
@@ -149,29 +160,66 @@ class RequestOutput:
160
161
outputs = []
162
include_prompt = True
152
- for seq in top_n_seqs:
163
+ for i, seq in enumerate(top_n_seqs):
164
output_text = seq.get_output_text_to_return(
165
text_buffer_length, delta)
166
+
167
output_token_ids = seq.get_output_token_ids_to_return(delta)
168
+ num_output_tokens = 1 if isinstance(output_token_ids,
169
+ int) else len(output_token_ids)
170
+
171
output_logprobs = seq.output_logprobs if include_logprobs else None
172
173
if delta:
174
# Slice logprobs delta if applicable
175
if output_logprobs:
161
- output_logprobs = output_logprobs[-len(output_token_ids):]
176
+ output_logprobs = output_logprobs[-num_output_tokens:]
177
# Don't include prompt if this is after the first output
178
# containing decode token ids
164
- if include_prompt and seq.get_output_len() > len(
165
- output_token_ids):
179
+ if include_prompt and seq.get_output_len() > num_output_tokens:
180
include_prompt = False
181
168
- outputs.append(
169
- CompletionOutput(
170
- seqs.index(seq), output_text, output_token_ids,
182
+ if use_cache:
183
+ # Get cached output object
184
+ cached_outputs = seq_group.cached_request_output.outputs # type: ignore
185
+ if i >= len(cached_outputs):
186
+ cached_outputs.append(
187
+ CompletionOutput(index=i,
188
+ text="",
189
+ token_ids=[],
190
+ cumulative_logprob=None,
191
+ logprobs=None,
192
+ finish_reason=None,
193
+ stop_reason=None))
194
+ output = cached_outputs[i]
195
+
196
+ # Init cached output object
197
+ assert output.index == i
198
+ output.text = output_text
199
+
200
+ if isinstance(output_token_ids, int):
201
+ output.token_ids.clear()
202
+ output.token_ids.append(output_token_ids)
203
+ else:
204
+ output.token_ids = output_token_ids
205
+
206
+ output.cumulative_logprob = seq.get_cumulative_logprob() \
207
+ if include_logprobs else None
208
+ output.logprobs = output_logprobs
209
+ output.finish_reason = SequenceStatus.get_finished_reason(
210
+ seq.status)
211
+ output.stop_reason = seq.stop_reason
212
+
213
+ else:
214
+ output = CompletionOutput(
215
+ seqs.index(seq), output_text, [output_token_ids]
216
+ if isinstance(output_token_ids, int) else output_token_ids,
217
seq.get_cumulative_logprob() if include_logprobs else None,
218
output_logprobs,
219
SequenceStatus.get_finished_reason(seq.status),
174
- seq.stop_reason))
220
+ seq.stop_reason)
221
+
222
+ outputs.append(output)
223
224
# Every sequence in the sequence group should have the same prompt.
225
if include_prompt:
@@ -188,16 +236,20 @@ class RequestOutput:
236
prompt_logprobs = None
237
finished_time = time.time() if finished else None
238
seq_group.set_finished_time(finished_time)
191
- return cls(seq_group.request_id,
192
- prompt,
193
- prompt_token_ids,
194
- prompt_logprobs,
195
- outputs,
196
- finished,
197
- seq_group.metrics,
198
- lora_request=seq_group.lora_request,
199
- encoder_prompt=encoder_prompt,
200
- encoder_prompt_token_ids=encoder_prompt_token_ids)
239
+
240
+ init_args = (seq_group.request_id, prompt, prompt_token_ids,
241
+ prompt_logprobs, outputs, finished, seq_group.metrics,
242
+ seq_group.lora_request, encoder_prompt,
243
+ encoder_prompt_token_ids)
244
+
245
+ if use_cache:
246
+ request_output = seq_group.cached_request_output
247
+ request_output.__init__(*init_args) # type: ignore
248
+
249
+ else:
250
+ request_output = cls(*init_args)
251
+
252
+ return request_output
253
254
def __repr__(self) -> str:
255
return (f"RequestOutput(request_id={self.request_id}, "
@@ -261,10 +313,10 @@ class EmbeddingRequestOutput:
313
class RequestOutputFactory:
314
315
@staticmethod
264
- def create(seq_group):
316
+ def create(seq_group: SequenceGroup, use_cache: bool = False):
317
# Determine the type based on a condition, for example:
318
if hasattr(seq_group,
319
'embeddings') and seq_group.embeddings is not None:
320
return EmbeddingRequestOutput.from_seq_group(seq_group)
321
else:
270
- return RequestOutput.from_seq_group(seq_group)
322
+ return RequestOutput.from_seq_group(seq_group, use_cache)
vllm/sequence.py
+19
-9
@@ -436,7 +436,7 @@ class Sequence:
436
self.stop_reason: Union[int, str, None] = None
437
438
# These are used to keep track of delta outputs
439
- self._last_token_ids_offset: int = 0
439
+ self._last_output_token_ids_offset: int = 0
440
self._last_output_text_offset: int = 0
441
442
# Used for incremental detokenization
@@ -499,18 +499,26 @@ class Sequence:
499
return self.output_text[last_offset:length]
500
return ""
501
502
- def get_output_token_ids_to_return(self,
503
- delta: bool) -> GenericSequence[int]:
502
+ def get_output_token_ids_to_return(
503
+ self, delta: bool) -> Union[GenericSequence[int], int]:
504
"""If delta is True, only new tokens since the last call to
505
this method are returned"""
506
if not delta:
507
return self.get_output_token_ids()
508
- length = self.get_output_len()
509
- last_offset = self._last_token_ids_offset
510
- if last_offset < length:
511
- self._last_token_ids_offset = length
512
- return self.data._output_token_ids[last_offset:]
513
- return ()
508
+
509
+ output_len = self.get_output_len()
510
+
511
+ # Get the number of new tokens
512
+ num_new_tokens = output_len - self._last_output_token_ids_offset
513
+ self._last_output_token_ids_offset = output_len
514
+
515
+ # Return new tokens
516
+ if num_new_tokens == 1:
517
+ # Optimization for single decode token case
518
+ # (which is what we have most of the time)
519
+ return self.data._cached_all_token_ids[-1]
520
+
521
+ return self.data._cached_all_token_ids[-num_new_tokens:]
522
523
def hash_of_block(self, logical_idx: int) -> int:
524
# TODO This can produce incorrect hash when block size > prompt size
@@ -671,6 +679,8 @@ class SequenceGroup:
679
self.encoder_seq = encoder_seq
680
self.trace_headers = trace_headers
681
682
+ self.cached_request_output = None
683
+
684
@property
685
def prompt(self) -> Optional[str]:
686
# All sequences in the group should have the same prompt.