@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a2aef3e59
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

Add output streaming support to multi-step + async while ensuring RequestOutput obj reuse (#8335)

Alexander Matveev committed Sep 23, 2024 at 18:38 UTC 1a2aef3e59f5429299618bd3b242833cb377f554
7 files changed +142 -42
tests/entrypoints/openai/test_accuracy.py
+5 -1
@@ -19,7 +19,11 @@ FILTER = "exact_match,strict-match"
19 RTOL = 0.03
20 EXPECTED_VALUE = 0.58
21 DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
22 -MORE_ARGS_LIST = [["--enable-chunked-prefill"], ["--num-scheduler-steps", "8"]]
22 +MORE_ARGS_LIST = [
23 + ["--enable-chunked-prefill"], # Chunked
24 + ["--num-scheduler-steps", "8"], # MS
25 + ["--num-scheduler-steps", "8", "--multi-step-stream-outputs"] # MS+Stream
26 +]
27
28
29 @pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
vllm/config.py
+2
@@ -960,6 +960,7 @@ class SchedulerConfig:
960 is_multimodal_model: bool = False,
961 preemption_mode: Optional[str] = None,
962 num_scheduler_steps: int = 1,
963 + multi_step_stream_outputs: bool = False,
964 send_delta_data: bool = False) -> None:
965 if max_num_batched_tokens is None:
966 if enable_chunked_prefill:
@@ -1000,6 +1001,7 @@ class SchedulerConfig:
1001 self.embedding_mode = embedding_mode
1002 self.preemption_mode = preemption_mode
1003 self.num_scheduler_steps = num_scheduler_steps
1004 + self.multi_step_stream_outputs = multi_step_stream_outputs
1005 self.send_delta_data = send_delta_data
1006 self._verify_args()
1007
vllm/engine/arg_utils.py
+6
@@ -145,6 +145,7 @@ class EngineArgs:
145 max_cpu_loras: Optional[int] = None
146 device: str = 'auto'
147 num_scheduler_steps: int = 1
148 + multi_step_stream_outputs: bool = False
149 ray_workers_use_nsight: bool = False
150 num_gpu_blocks_override: Optional[int] = None
151 num_lookahead_slots: int = 0
@@ -595,6 +596,10 @@ class EngineArgs:
596 help=('Maximum number of forward steps per '
597 'scheduler call.'))
598
599 + parser.add_argument(
600 + '--multi-step-stream-outputs',
601 + action='store_true',
602 + help='If True, then multi-step will stream outputs for every step')
603 parser.add_argument(
604 '--scheduler-delay-factor',
605 type=float,
@@ -999,6 +1004,7 @@ class EngineArgs:
1004 is_multimodal_model=model_config.is_multimodal_model,
1005 preemption_mode=self.preemption_mode,
1006 num_scheduler_steps=self.num_scheduler_steps,
1007 + multi_step_stream_outputs=self.multi_step_stream_outputs,
1008 send_delta_data=(envs.VLLM_USE_RAY_SPMD_WORKER
1009 and parallel_config.use_ray),
1010 )
vllm/engine/llm_engine.py
+28 -9
@@ -95,7 +95,7 @@ class OutputData(NamedTuple):
95
96 class SchedulerContext:
97
98 - def __init__(self):
98 + def __init__(self, multi_step_stream_outputs: bool = False):
99 self.output_queue: Deque[OutputData] = deque()
100 self.request_outputs: List[Union[RequestOutput,
101 EmbeddingRequestOutput]] = []
@@ -103,6 +103,8 @@ class SchedulerContext:
103 List[SequenceGroupMetadata]] = None
104 self.scheduler_outputs: Optional[SchedulerOutputs] = None
105
106 + self.multi_step_stream_outputs: bool = multi_step_stream_outputs
107 +
108 def append_output(self, outputs: List[SamplerOutput],
109 seq_group_metadata_list: List[SequenceGroupMetadata],
110 scheduler_outputs: SchedulerOutputs, is_async: bool,
@@ -219,6 +221,7 @@ class LLMEngine:
221 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
222 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
223 input_registry: InputRegistry = INPUT_REGISTRY,
224 + use_cached_outputs: bool = False,
225 ) -> None:
226 logger.info(
227 "Initializing an LLM engine (v%s) with config: "
@@ -234,8 +237,9 @@ class LLMEngine:
237 "quantization_param_path=%s, device_config=%s, "
238 "decoding_config=%r, observability_config=%r, "
239 "seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
237 - "num_scheduler_steps=%d, enable_prefix_caching=%s, "
238 - "use_async_output_proc=%s, mm_processor_kwargs=%s)",
240 + "num_scheduler_steps=%d, multi_step_stream_outputs=%s, "
241 + "enable_prefix_caching=%s, use_async_output_proc=%s, "
242 + "use_cached_outputs=%s, mm_processor_kwargs=%s)",
243 VLLM_VERSION,
244 model_config.model,
245 speculative_config,
@@ -266,8 +270,10 @@ class LLMEngine:
270 model_config.served_model_name,
271 scheduler_config.use_v2_block_manager,
272 scheduler_config.num_scheduler_steps,
273 + scheduler_config.multi_step_stream_outputs,
274 cache_config.enable_prefix_caching,
275 model_config.use_async_output_proc,
276 + use_cached_outputs,
277 model_config.mm_processor_kwargs,
278 )
279 # TODO(woosuk): Print more configs in debug mode.
@@ -287,6 +293,7 @@ class LLMEngine:
293 self.observability_config = observability_config or ObservabilityConfig(
294 )
295 self.log_stats = log_stats
296 + self.use_cached_outputs = use_cached_outputs
297
298 if not self.model_config.skip_tokenizer_init:
299 self.tokenizer = self._init_tokenizer()
@@ -379,7 +386,8 @@ class LLMEngine:
386 ]
387
388 self.scheduler_contexts = [
382 - SchedulerContext()
389 + SchedulerContext(multi_step_stream_outputs=self.scheduler_config.
390 + multi_step_stream_outputs)
391 for _ in range(self.parallel_config.pipeline_parallel_size)
392 ]
393
@@ -998,7 +1006,8 @@ class LLMEngine:
1006
1007 seq_group = scheduled_seq_group.seq_group
1008 seq_group.maybe_set_first_token_time(now)
1001 - request_output = RequestOutputFactory.create(seq_group)
1009 + request_output = RequestOutputFactory.create(
1010 + seq_group, use_cache=self.use_cached_outputs)
1011 if request_output:
1012 ctx.request_outputs.append(request_output)
1013
@@ -1019,8 +1028,8 @@ class LLMEngine:
1028 for scheduler in self.scheduler:
1029 scheduler.free_finished_seq_groups()
1030
1022 - # For multi-step, do not create outputs each iteration
1023 - if not is_last_step:
1031 + # For multi-step without streaming, don't create outputs each iteration
1032 + if not is_last_step and not ctx.multi_step_stream_outputs:
1033 # Immediately process request outputs here (if callback is given)
1034 if (finished_now
1035 and self.process_request_outputs_callback is not None):
@@ -1037,17 +1046,27 @@ class LLMEngine:
1046
1047 seq_group = scheduled_seq_group.seq_group
1048 seq_group.maybe_set_first_token_time(now)
1040 - request_output = RequestOutputFactory.create(seq_group)
1049 + request_output = RequestOutputFactory.create(
1050 + seq_group, use_cache=self.use_cached_outputs)
1051 if request_output:
1052 ctx.request_outputs.append(request_output)
1053
1054 + # For multi-step with streaming, create outputs each iteration
1055 + if not is_last_step and ctx.multi_step_stream_outputs:
1056 + # Immediately process request outputs here (if callback is given)
1057 + if self.process_request_outputs_callback is not None:
1058 + self.process_request_outputs_callback(ctx.request_outputs)
1059 + ctx.request_outputs.clear()
1060 + return
1061 +
1062 for seq_group in scheduler_outputs.ignored_seq_groups:
1063 params = seq_group.sampling_params
1064 if params is not None and params.output_kind == (
1065 RequestOutputKind.DELTA) and not seq_group.is_finished():
1066 continue
1067
1050 - request_output = RequestOutputFactory.create(seq_group)
1068 + request_output = RequestOutputFactory.create(
1069 + seq_group, use_cache=self.use_cached_outputs)
1070 if request_output:
1071 ctx.request_outputs.append(request_output)
1072
vllm/engine/multiprocessing/engine.py
+8 -1
@@ -66,7 +66,14 @@ class MQLLMEngine:
66 *args,
67 log_requests: bool = True,
68 **kwargs) -> None:
69 - self.engine = LLMEngine(*args, **kwargs)
69 + # For MQLLMEngine, we can use cached outputs, since each new request
70 + # output is immediately pickled and send over the socket, which frees
71 + # the python object to be reused again.
72 + use_cached_outputs = True
73 +
74 + self.engine = LLMEngine(*args,
75 + **kwargs,
76 + use_cached_outputs=use_cached_outputs)
77 self.log_requests = log_requests
78
79 self.use_async_sockets = use_async_sockets
vllm/outputs.py
+74 -22
@@ -114,17 +114,28 @@ class RequestOutput:
114 self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116 @classmethod
117 - def from_seq_group(cls,
118 - seq_group: SequenceGroup) -> Optional["RequestOutput"]:
117 + def from_seq_group(cls, seq_group: SequenceGroup,
118 + use_cache: bool) -> Optional["RequestOutput"]:
119 sampling_params = seq_group.sampling_params
120 if sampling_params is None:
121 raise ValueError(
122 "Sampling parameters are missing for a CompletionRequest.")
123 +
124 finished = seq_group.is_finished()
125 if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
126 not finished):
127 return None
128
129 + # Init cache (if needed)
130 + if use_cache and seq_group.cached_request_output is None:
131 + seq_group.cached_request_output = RequestOutput( # type: ignore
132 + request_id="",
133 + prompt=None,
134 + prompt_token_ids=[],
135 + prompt_logprobs=None,
136 + outputs=[],
137 + finished=False)
138 +
139 seqs = seq_group.get_seqs()
140 if len(seqs) == 1:
141 top_n_seqs = seqs
@@ -149,29 +160,66 @@ class RequestOutput:
160
161 outputs = []
162 include_prompt = True
152 - for seq in top_n_seqs:
163 + for i, seq in enumerate(top_n_seqs):
164 output_text = seq.get_output_text_to_return(
165 text_buffer_length, delta)
166 +
167 output_token_ids = seq.get_output_token_ids_to_return(delta)
168 + num_output_tokens = 1 if isinstance(output_token_ids,
169 + int) else len(output_token_ids)
170 +
171 output_logprobs = seq.output_logprobs if include_logprobs else None
172
173 if delta:
174 # Slice logprobs delta if applicable
175 if output_logprobs:
161 - output_logprobs = output_logprobs[-len(output_token_ids):]
176 + output_logprobs = output_logprobs[-num_output_tokens:]
177 # Don't include prompt if this is after the first output
178 # containing decode token ids
164 - if include_prompt and seq.get_output_len() > len(
165 - output_token_ids):
179 + if include_prompt and seq.get_output_len() > num_output_tokens:
180 include_prompt = False
181
168 - outputs.append(
169 - CompletionOutput(
170 - seqs.index(seq), output_text, output_token_ids,
182 + if use_cache:
183 + # Get cached output object
184 + cached_outputs = seq_group.cached_request_output.outputs # type: ignore
185 + if i >= len(cached_outputs):
186 + cached_outputs.append(
187 + CompletionOutput(index=i,
188 + text="",
189 + token_ids=[],
190 + cumulative_logprob=None,
191 + logprobs=None,
192 + finish_reason=None,
193 + stop_reason=None))
194 + output = cached_outputs[i]
195 +
196 + # Init cached output object
197 + assert output.index == i
198 + output.text = output_text
199 +
200 + if isinstance(output_token_ids, int):
201 + output.token_ids.clear()
202 + output.token_ids.append(output_token_ids)
203 + else:
204 + output.token_ids = output_token_ids
205 +
206 + output.cumulative_logprob = seq.get_cumulative_logprob() \
207 + if include_logprobs else None
208 + output.logprobs = output_logprobs
209 + output.finish_reason = SequenceStatus.get_finished_reason(
210 + seq.status)
211 + output.stop_reason = seq.stop_reason
212 +
213 + else:
214 + output = CompletionOutput(
215 + seqs.index(seq), output_text, [output_token_ids]
216 + if isinstance(output_token_ids, int) else output_token_ids,
217 seq.get_cumulative_logprob() if include_logprobs else None,
218 output_logprobs,
219 SequenceStatus.get_finished_reason(seq.status),
174 - seq.stop_reason))
220 + seq.stop_reason)
221 +
222 + outputs.append(output)
223
224 # Every sequence in the sequence group should have the same prompt.
225 if include_prompt:
@@ -188,16 +236,20 @@ class RequestOutput:
236 prompt_logprobs = None
237 finished_time = time.time() if finished else None
238 seq_group.set_finished_time(finished_time)
191 - return cls(seq_group.request_id,
192 - prompt,
193 - prompt_token_ids,
194 - prompt_logprobs,
195 - outputs,
196 - finished,
197 - seq_group.metrics,
198 - lora_request=seq_group.lora_request,
199 - encoder_prompt=encoder_prompt,
200 - encoder_prompt_token_ids=encoder_prompt_token_ids)
239 +
240 + init_args = (seq_group.request_id, prompt, prompt_token_ids,
241 + prompt_logprobs, outputs, finished, seq_group.metrics,
242 + seq_group.lora_request, encoder_prompt,
243 + encoder_prompt_token_ids)
244 +
245 + if use_cache:
246 + request_output = seq_group.cached_request_output
247 + request_output.__init__(*init_args) # type: ignore
248 +
249 + else:
250 + request_output = cls(*init_args)
251 +
252 + return request_output
253
254 def __repr__(self) -> str:
255 return (f"RequestOutput(request_id={self.request_id}, "
@@ -261,10 +313,10 @@ class EmbeddingRequestOutput:
313 class RequestOutputFactory:
314
315 @staticmethod
264 - def create(seq_group):
316 + def create(seq_group: SequenceGroup, use_cache: bool = False):
317 # Determine the type based on a condition, for example:
318 if hasattr(seq_group,
319 'embeddings') and seq_group.embeddings is not None:
320 return EmbeddingRequestOutput.from_seq_group(seq_group)
321 else:
270 - return RequestOutput.from_seq_group(seq_group)
322 + return RequestOutput.from_seq_group(seq_group, use_cache)
vllm/sequence.py
+19 -9
@@ -436,7 +436,7 @@ class Sequence:
436 self.stop_reason: Union[int, str, None] = None
437
438 # These are used to keep track of delta outputs
439 - self._last_token_ids_offset: int = 0
439 + self._last_output_token_ids_offset: int = 0
440 self._last_output_text_offset: int = 0
441
442 # Used for incremental detokenization
@@ -499,18 +499,26 @@ class Sequence:
499 return self.output_text[last_offset:length]
500 return ""
501
502 - def get_output_token_ids_to_return(self,
503 - delta: bool) -> GenericSequence[int]:
502 + def get_output_token_ids_to_return(
503 + self, delta: bool) -> Union[GenericSequence[int], int]:
504 """If delta is True, only new tokens since the last call to
505 this method are returned"""
506 if not delta:
507 return self.get_output_token_ids()
508 - length = self.get_output_len()
509 - last_offset = self._last_token_ids_offset
510 - if last_offset < length:
511 - self._last_token_ids_offset = length
512 - return self.data._output_token_ids[last_offset:]
513 - return ()
508 +
509 + output_len = self.get_output_len()
510 +
511 + # Get the number of new tokens
512 + num_new_tokens = output_len - self._last_output_token_ids_offset
513 + self._last_output_token_ids_offset = output_len
514 +
515 + # Return new tokens
516 + if num_new_tokens == 1:
517 + # Optimization for single decode token case
518 + # (which is what we have most of the time)
519 + return self.data._cached_all_token_ids[-1]
520 +
521 + return self.data._cached_all_token_ids[-num_new_tokens:]
522
523 def hash_of_block(self, logical_idx: int) -> int:
524 # TODO This can produce incorrect hash when block size > prompt size
@@ -671,6 +679,8 @@ class SequenceGroup:
679 self.encoder_seq = encoder_seq
680 self.trace_headers = trace_headers
681
682 + self.cached_request_output = None
683 +
684 @property
685 def prompt(self) -> Optional[str]:
686 # All sequences in the group should have the same prompt.