@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6da1ab6b41
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Core] Adding Priority Scheduling (#5958)

Archit Patke committed Sep 24, 2024 at 21:50 UTC 6da1ab6b4134d76391a0c31a048e5d04b6283769
6 files changed +410 -8
benchmarks/benchmark_prioritization.py new
+295
@@ -0,0 +1,295 @@
1 +"""Benchmark offline prioritization."""
2 +import argparse
3 +import json
4 +import random
5 +import time
6 +from typing import List, Optional, Tuple
7 +
8 +from transformers import AutoTokenizer, PreTrainedTokenizerBase
9 +
10 +from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 +
12 +
13 +def sample_requests(
14 + dataset_path: str,
15 + num_requests: int,
16 + tokenizer: PreTrainedTokenizerBase,
17 + fixed_output_len: Optional[int],
18 +) -> List[Tuple[str, int, int]]:
19 + if fixed_output_len is not None and fixed_output_len < 4:
20 + raise ValueError("output_len too small")
21 +
22 + # Load the dataset.
23 + with open(dataset_path) as f:
24 + dataset = json.load(f)
25 + # Filter out the conversations with less than 2 turns.
26 + dataset = [data for data in dataset if len(data["conversations"]) >= 2]
27 + # Only keep the first two turns of each conversation.
28 + dataset = [(data["conversations"][0]["value"],
29 + data["conversations"][1]["value"]) for data in dataset]
30 +
31 + # Shuffle the dataset.
32 + random.shuffle(dataset)
33 +
34 + # Filter out sequences that are too long or too short
35 + filtered_dataset: List[Tuple[str, int, int]] = []
36 + for i in range(len(dataset)):
37 + if len(filtered_dataset) == num_requests:
38 + break
39 +
40 + # Tokenize the prompts and completions.
41 + prompt = dataset[i][0]
42 + prompt_token_ids = tokenizer(prompt).input_ids
43 + completion = dataset[i][1]
44 + completion_token_ids = tokenizer(completion).input_ids
45 + prompt_len = len(prompt_token_ids)
46 + output_len = len(completion_token_ids
47 + ) if fixed_output_len is None else fixed_output_len
48 + if prompt_len < 4 or output_len < 4:
49 + # Prune too short sequences.
50 + continue
51 + if prompt_len > 1024 or prompt_len + output_len > 2048:
52 + # Prune too long sequences.
53 + continue
54 +
55 + #Select a equi-probable random priority
56 + priority = 0 if random.random() < 0.5 else 1
57 +
58 + filtered_dataset.append((prompt, prompt_len, output_len, priority))
59 +
60 + return filtered_dataset
61 +
62 +
63 +def run_vllm(
64 + requests: List[Tuple[str, int, int]],
65 + model: str,
66 + tokenizer: str,
67 + quantization: Optional[str],
68 + tensor_parallel_size: int,
69 + seed: int,
70 + n: int,
71 + use_beam_search: bool,
72 + trust_remote_code: bool,
73 + dtype: str,
74 + max_model_len: Optional[int],
75 + enforce_eager: bool,
76 + kv_cache_dtype: str,
77 + quantization_param_path: Optional[str],
78 + device: str,
79 + enable_prefix_caching: bool,
80 + enable_chunked_prefill: bool,
81 + max_num_batched_tokens: int,
82 + gpu_memory_utilization: float = 0.9,
83 + download_dir: Optional[str] = None,
84 +) -> float:
85 + from vllm import LLM, SamplingParams
86 + llm = LLM(
87 + model=model,
88 + tokenizer=tokenizer,
89 + quantization=quantization,
90 + tensor_parallel_size=tensor_parallel_size,
91 + seed=seed,
92 + trust_remote_code=trust_remote_code,
93 + dtype=dtype,
94 + max_model_len=max_model_len,
95 + gpu_memory_utilization=gpu_memory_utilization,
96 + enforce_eager=enforce_eager,
97 + kv_cache_dtype=kv_cache_dtype,
98 + quantization_param_path=quantization_param_path,
99 + device=device,
100 + enable_prefix_caching=enable_prefix_caching,
101 + download_dir=download_dir,
102 + enable_chunked_prefill=enable_chunked_prefill,
103 + max_num_batched_tokens=max_num_batched_tokens,
104 + disable_log_stats=False,
105 + )
106 +
107 + # Add the requests to the engine.
108 + prompts = []
109 + sampling_params = []
110 + priority = []
111 + for prompt, _, output_len, _priority in requests:
112 + prompts.append(prompt)
113 + priority.append(_priority)
114 + sampling_params.append(
115 + SamplingParams(
116 + n=n,
117 + temperature=0.0 if use_beam_search else 1.0,
118 + top_p=1.0,
119 + use_beam_search=use_beam_search,
120 + ignore_eos=True,
121 + max_tokens=output_len,
122 + ))
123 +
124 + start = time.perf_counter()
125 + llm.generate(prompts, sampling_params, priority=priority, use_tqdm=True)
126 + end = time.perf_counter()
127 + return end - start
128 +
129 +
130 +def main(args: argparse.Namespace):
131 + print(args)
132 + random.seed(args.seed)
133 +
134 + # Sample the requests.
135 + tokenizer = AutoTokenizer.from_pretrained(
136 + args.tokenizer, trust_remote_code=args.trust_remote_code)
137 + if args.dataset is None:
138 + # Synthesize a prompt with the given input length.
139 + prompt = "hi" * (args.input_len - 1)
140 + requests = [(prompt, args.input_len, args.output_len)
141 + for _ in range(args.num_prompts)]
142 + else:
143 + requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
144 + args.output_len)
145 +
146 + if args.backend == "vllm":
147 + elapsed_time = run_vllm(
148 + requests, args.model, args.tokenizer, args.quantization,
149 + args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
150 + args.trust_remote_code, args.dtype, args.max_model_len,
151 + args.enforce_eager, args.kv_cache_dtype,
152 + args.quantization_param_path, args.device,
153 + args.enable_prefix_caching, args.enable_chunked_prefill,
154 + args.max_num_batched_tokens, args.gpu_memory_utilization,
155 + args.download_dir)
156 + else:
157 + raise ValueError(f"Unknown backend: {args.backend}")
158 + total_num_tokens = sum(prompt_len + output_len
159 + for _, prompt_len, output_len, priority in requests)
160 + print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
161 + f"{total_num_tokens / elapsed_time:.2f} tokens/s")
162 +
163 + # Output JSON results if specified
164 + if args.output_json:
165 + results = {
166 + "elapsed_time": elapsed_time,
167 + "num_requests": len(requests),
168 + "total_num_tokens": total_num_tokens,
169 + "requests_per_second": len(requests) / elapsed_time,
170 + "tokens_per_second": total_num_tokens / elapsed_time,
171 + }
172 + with open(args.output_json, "w") as f:
173 + json.dump(results, f, indent=4)
174 +
175 +
176 +if __name__ == "__main__":
177 + parser = argparse.ArgumentParser(description="Benchmark the throughput.")
178 + parser.add_argument("--backend",
179 + type=str,
180 + choices=["vllm", "hf", "mii"],
181 + default="vllm")
182 + parser.add_argument("--dataset",
183 + type=str,
184 + default=None,
185 + help="Path to the dataset.")
186 + parser.add_argument("--input-len",
187 + type=int,
188 + default=None,
189 + help="Input prompt length for each request")
190 + parser.add_argument("--output-len",
191 + type=int,
192 + default=None,
193 + help="Output length for each request. Overrides the "
194 + "output length from the dataset.")
195 + parser.add_argument("--model", type=str, default="facebook/opt-125m")
196 + parser.add_argument("--tokenizer", type=str, default=None)
197 + parser.add_argument('--quantization',
198 + '-q',
199 + choices=[*QUANTIZATION_METHODS, None],
200 + default=None)
201 + parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
202 + parser.add_argument("--n",
203 + type=int,
204 + default=1,
205 + help="Number of generated sequences per prompt.")
206 + parser.add_argument("--use-beam-search", action="store_true")
207 + parser.add_argument("--num-prompts",
208 + type=int,
209 + default=200,
210 + help="Number of prompts to process.")
211 + parser.add_argument("--seed", type=int, default=0)
212 + parser.add_argument('--trust-remote-code',
213 + action='store_true',
214 + help='trust remote code from huggingface')
215 + parser.add_argument(
216 + '--max-model-len',
217 + type=int,
218 + default=None,
219 + help='Maximum length of a sequence (including prompt and output). '
220 + 'If None, will be derived from the model.')
221 + parser.add_argument(
222 + '--dtype',
223 + type=str,
224 + default='auto',
225 + choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
226 + help='data type for model weights and activations. '
227 + 'The "auto" option will use FP16 precision '
228 + 'for FP32 and FP16 models, and BF16 precision '
229 + 'for BF16 models.')
230 + parser.add_argument('--gpu-memory-utilization',
231 + type=float,
232 + default=0.9,
233 + help='the fraction of GPU memory to be used for '
234 + 'the model executor, which can range from 0 to 1.'
235 + 'If unspecified, will use the default value of 0.9.')
236 + parser.add_argument("--enforce-eager",
237 + action="store_true",
238 + help="enforce eager execution")
239 + parser.add_argument(
240 + '--kv-cache-dtype',
241 + type=str,
242 + choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
243 + default="auto",
244 + help='Data type for kv cache storage. If "auto", will use model '
245 + 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
246 + 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
247 + parser.add_argument(
248 + '--quantization-param-path',
249 + type=str,
250 + default=None,
251 + help='Path to the JSON file containing the KV cache scaling factors. '
252 + 'This should generally be supplied, when KV cache dtype is FP8. '
253 + 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
254 + 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
255 + 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
256 + 'instead supported for common inference criteria.')
257 + parser.add_argument(
258 + "--device",
259 + type=str,
260 + default="cuda",
261 + choices=["cuda", "cpu"],
262 + help='device type for vLLM execution, supporting CUDA and CPU.')
263 + parser.add_argument(
264 + "--enable-prefix-caching",
265 + action='store_true',
266 + help="enable automatic prefix caching for vLLM backend.")
267 + parser.add_argument("--enable-chunked-prefill",
268 + action='store_true',
269 + help="enable chunked prefill for vLLM backend.")
270 + parser.add_argument('--max-num-batched-tokens',
271 + type=int,
272 + default=None,
273 + help='maximum number of batched tokens per '
274 + 'iteration')
275 + parser.add_argument('--download-dir',
276 + type=str,
277 + default=None,
278 + help='directory to download and load the weights, '
279 + 'default to the default cache dir of huggingface')
280 + parser.add_argument(
281 + '--output-json',
282 + type=str,
283 + default=None,
284 + help='Path to save the throughput results in JSON format.')
285 +
286 + args = parser.parse_args()
287 + if args.tokenizer is None:
288 + args.tokenizer = args.model
289 + if args.dataset is None:
290 + assert args.input_len is not None
291 + assert args.output_len is not None
292 + else:
293 + assert args.input_len is None
294 +
295 + main(args)
vllm/config.py
+4 -2
@@ -961,7 +961,7 @@ class SchedulerConfig:
961 workers instead of an entire data. It should be enabled only
962 when SPMD worker architecture is enabled. I.e.,
963 VLLM_USE_RAY_SPMD_WORKER=1
964 -
964 + policy: The scheduling policy to use. "fcfs" (default) or "priority".
965 """
966
967 def __init__(self,
@@ -977,7 +977,8 @@ class SchedulerConfig:
977 preemption_mode: Optional[str] = None,
978 num_scheduler_steps: int = 1,
979 multi_step_stream_outputs: bool = False,
980 - send_delta_data: bool = False) -> None:
980 + send_delta_data: bool = False,
981 + policy: str = "fcfs") -> None:
982 if max_num_batched_tokens is None:
983 if enable_chunked_prefill:
984 # It is the values that have the best balance between ITL
@@ -1019,6 +1020,7 @@ class SchedulerConfig:
1020 self.num_scheduler_steps = num_scheduler_steps
1021 self.multi_step_stream_outputs = multi_step_stream_outputs
1022 self.send_delta_data = send_delta_data
1023 + self.policy = policy
1024 self._verify_args()
1025
1026 def _verify_args(self) -> None:
vllm/core/scheduler.py
+77
@@ -766,6 +766,79 @@ class Scheduler:
766 else:
767 return prompt_limit
768
769 + def _get_priority(self,
770 + seq_group: SequenceGroup) -> Tuple[Optional[int], float]:
771 + """ Get the priority of the sequence group.
772 + Highest preference to user-defined priority, followed by arrival time.
773 + Args:
774 + seq_group: The sequence group input.
775 + Returns:
776 + The priority of the sequence group.
777 + """
778 + return seq_group.priority, seq_group.arrival_time
779 +
780 + def _schedule_priority_preemption(
781 + self,
782 + budget: SchedulingBudget,
783 + ) -> int:
784 + """Sorts waiting and running queue. Also, force preempt requests
785 + from the running queue if their priority is lower.
786 + Priority-based preemption is used with the priority policy.
787 + Args:
788 + budget: The scheduling budget. The argument is in-place updated
789 + when any requests are scheduled.
790 + Returns:
791 + A count of priority-based preemptions.
792 + """
793 +
794 + waiting_queue = self.waiting
795 +
796 + running_queue = deque(sorted(self.running, key=self._get_priority))
797 +
798 + blocks_to_swap_out: List[Tuple[int, int]] = []
799 + force_preemption_count = 0
800 +
801 + if waiting_queue:
802 + seq_group = waiting_queue.popleft()
803 + num_new_seqs = seq_group.get_max_num_running_seqs()
804 + num_new_tokens = self._get_num_new_tokens(seq_group,
805 + SequenceStatus.WAITING,
806 + False, budget)
807 +
808 + #Only preempt if priority inversion exists
809 + while running_queue and self._get_priority(
810 + running_queue[-1]) > self._get_priority(seq_group):
811 + #Only preempt if waiting sequence cannot be allocated
812 + can_allocate = self.block_manager.can_allocate(seq_group)
813 + if (num_new_tokens and can_allocate == AllocStatus.OK
814 + and budget.can_schedule(num_new_tokens=num_new_tokens,
815 + num_new_seqs=num_new_seqs)):
816 + break
817 +
818 + #Adjust budget to remove the victim sequence group
819 + vseq_group = running_queue.pop()
820 + num_running_tokens = self._get_num_new_tokens(
821 + vseq_group, SequenceStatus.RUNNING, False, budget)
822 + budget.subtract_num_batched_tokens(vseq_group.request_id,
823 + num_running_tokens)
824 + num_running_seqs = vseq_group.get_max_num_running_seqs()
825 + budget.subtract_num_seqs(vseq_group.request_id,
826 + num_running_seqs)
827 +
828 + #Preempt out the victim sequence group
829 + self._preempt(vseq_group, blocks_to_swap_out,
830 + PreemptionMode.RECOMPUTE)
831 + waiting_queue.appendleft(vseq_group)
832 + force_preemption_count += 1
833 + #Put the sequence back into the waiting queue
834 + waiting_queue.appendleft(seq_group)
835 +
836 + waiting_queue = deque(sorted(waiting_queue, key=self._get_priority))
837 +
838 + self.waiting = waiting_queue
839 + self.running = running_queue
840 + return force_preemption_count
841 +
842 def _schedule_prefills(
843 self,
844 budget: SchedulingBudget,
@@ -917,6 +990,10 @@ class Scheduler:
990 curr_loras,
991 enable_chunking=False)
992
993 + if len(prefills.seq_groups
994 + ) == 0 and self.scheduler_config.policy == "priority":
995 + self._schedule_priority_preemption(budget)
996 +
997 # Don't schedule decodes if prefills are scheduled.
998 # NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
999 # only contains decode requests, not chunked prefills.
vllm/engine/llm_engine.py
+20 -4
@@ -631,6 +631,7 @@ class LLMEngine:
631 lora_request: Optional[LoRARequest],
632 prompt_adapter_request: Optional[PromptAdapterRequest],
633 trace_headers: Optional[Mapping[str, str]] = None,
634 + priority: int = 0,
635 ) -> None:
636 self._validate_model_inputs(processed_inputs)
637 # Create the sequences.
@@ -661,7 +662,8 @@ class LLMEngine:
662 lora_request=lora_request,
663 trace_headers=trace_headers,
664 prompt_adapter_request=prompt_adapter_request,
664 - encoder_seq=encoder_seq)
665 + encoder_seq=encoder_seq,
666 + priority=priority)
667 elif isinstance(params, PoolingParams):
668 seq_group = self._create_sequence_group_with_pooling(
669 request_id,
@@ -670,7 +672,8 @@ class LLMEngine:
672 arrival_time=arrival_time,
673 lora_request=lora_request,
674 prompt_adapter_request=prompt_adapter_request,
673 - encoder_seq=encoder_seq)
675 + encoder_seq=encoder_seq,
676 + priority=priority)
677 else:
678 raise ValueError(
679 "Either SamplingParams or PoolingParams must be provided.")
@@ -695,6 +698,7 @@ class LLMEngine:
698 lora_request: Optional[LoRARequest] = None,
699 trace_headers: Optional[Mapping[str, str]] = None,
700 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
701 + priority: int = 0,
702 ) -> None:
703 """Add a request to the engine's request pool.
704
@@ -713,6 +717,8 @@ class LLMEngine:
717 arrival_time: The arrival time of the request. If None, we use
718 the current monotonic time.
719 trace_headers: OpenTelemetry trace headers.
720 + priority: The priority of the request.
721 + Only applicable with priority scheduling.
722
723 Details:
724 - Set arrival_time to the current time if it is None.
@@ -741,6 +747,11 @@ class LLMEngine:
747 if lora_request is not None and not self.lora_config:
748 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
749 "not enabled!")
750 +
751 + if priority > 0 and not self.scheduler_config.policy == "priority":
752 + raise ValueError(f"Got priority {priority} but "
753 + "Priority scheduling is not enabled.")
754 +
755 if arrival_time is None:
756 arrival_time = time.time()
757
@@ -760,6 +771,7 @@ class LLMEngine:
771 lora_request=lora_request,
772 prompt_adapter_request=prompt_adapter_request,
773 trace_headers=trace_headers,
774 + priority=priority,
775 )
776
777 def _create_sequence_group_with_sampling(
@@ -772,6 +784,7 @@ class LLMEngine:
784 trace_headers: Optional[Mapping[str, str]] = None,
785 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
786 encoder_seq: Optional[Sequence] = None,
787 + priority: int = 0,
788 ) -> SequenceGroup:
789 """Creates a SequenceGroup with SamplingParams."""
790 max_logprobs = self.get_model_config().max_logprobs
@@ -798,7 +811,8 @@ class LLMEngine:
811 lora_request=lora_request,
812 trace_headers=trace_headers,
813 prompt_adapter_request=prompt_adapter_request,
801 - encoder_seq=encoder_seq)
814 + encoder_seq=encoder_seq,
815 + priority=priority)
816
817 return seq_group
818
@@ -811,6 +825,7 @@ class LLMEngine:
825 lora_request: Optional[LoRARequest],
826 prompt_adapter_request: Optional[PromptAdapterRequest],
827 encoder_seq: Optional[Sequence] = None,
828 + priority: int = 0,
829 ) -> SequenceGroup:
830 """Creates a SequenceGroup with PoolingParams."""
831 # Defensive copy of PoolingParams, which are used by the pooler
@@ -823,7 +838,8 @@ class LLMEngine:
838 lora_request=lora_request,
839 pooling_params=pooling_params,
840 prompt_adapter_request=prompt_adapter_request,
826 - encoder_seq=encoder_seq)
841 + encoder_seq=encoder_seq,
842 + priority=priority)
843 return seq_group
844
845 def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
vllm/entrypoints/llm.py
+10 -2
@@ -320,7 +320,8 @@ class LLM:
320 lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
321 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
322 guided_options_request: Optional[Union[LLMGuidedOptions,
323 - GuidedDecodingRequest]] = None
323 + GuidedDecodingRequest]] = None,
324 + priority: Optional[List[int]] = None,
325 ) -> List[RequestOutput]:
326 """Generates the completions for the input prompts.
327
@@ -339,6 +340,8 @@ class LLM:
340 lora_request: LoRA request to use for generation, if any.
341 prompt_adapter_request: Prompt Adapter request to use for
342 generation, if any.
343 + priority: The priority of the requests, if any.
344 + Only applicable when priority scheduling policy is enabled.
345
346 Returns:
347 A list of ``RequestOutput`` objects containing the
@@ -379,7 +382,8 @@ class LLM:
382 params=sampling_params,
383 lora_request=lora_request,
384 prompt_adapter_request=prompt_adapter_request,
382 - guided_options=guided_options_request)
385 + guided_options=guided_options_request,
386 + priority=priority)
387
388 outputs = self._run_engine(use_tqdm=use_tqdm)
389 return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -782,6 +786,7 @@ class LLM:
786 lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
787 prompt_adapter_request: Optional[PromptAdapterRequest],
788 guided_options: Optional[GuidedDecodingRequest] = None,
789 + priority: Optional[List[int]] = None,
790 ) -> None:
791 if isinstance(inputs, (str, dict)):
792 # Convert a single prompt to a list.
@@ -811,6 +816,7 @@ class LLM:
816 lora_request=lora_request[i] if isinstance(
817 lora_request, Sequence) else lora_request,
818 prompt_adapter_request=prompt_adapter_request,
819 + priority=priority[i] if priority else 0,
820 )
821
822 def _add_request(
@@ -819,6 +825,7 @@ class LLM:
825 params: Union[SamplingParams, PoolingParams],
826 lora_request: Optional[LoRARequest] = None,
827 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
828 + priority: int = 0,
829 ) -> None:
830 request_id = str(next(self.request_counter))
831 self.llm_engine.add_request(
@@ -827,6 +834,7 @@ class LLM:
834 params,
835 lora_request=lora_request,
836 prompt_adapter_request=prompt_adapter_request,
837 + priority=priority,
838 )
839
840 def _add_guided_processor(
vllm/sequence.py
+4
@@ -646,6 +646,7 @@ class SequenceGroup:
646 unless you are working with an encoder/decoder model.
647 trace_headers: OpenTelemetry trace headers.
648 prompt_adapter_request: Prompt Adapter request.
649 + priority: User-defined priority of the request.
650 """
651
652 def __init__(
@@ -660,9 +661,11 @@ class SequenceGroup:
661 encoder_seq: Optional[Sequence] = None,
662 trace_headers: Optional[Mapping[str, str]] = None,
663 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
664 + priority: int = 0,
665 ) -> None:
666 self.request_id = request_id
667 self.seqs = seqs
668 + self.arrival_time = arrival_time
669 self.is_single_seq = len(seqs) == 1
670 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
671
@@ -680,6 +683,7 @@ class SequenceGroup:
683 self.prompt_adapter_request = prompt_adapter_request
684 self.encoder_seq = encoder_seq
685 self.trace_headers = trace_headers
686 + self.priority = priority
687
688 self.cached_request_output = None
689