A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core] Adding Priority Scheduling (#5958)
Archit Patke committed
Sep 24, 2024 at 21:50 UTC
6da1ab6b4134d76391a0c31a048e5d04b6283769
6 files changed
+410
-8
benchmarks/benchmark_prioritization.py
new
+295
@@ -0,0 +1,295 @@
1
+"""Benchmark offline prioritization."""
2
+import argparse
3
+import json
4
+import random
5
+import time
6
+from typing import List, Optional, Tuple
7
+
8
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
9
+
10
+from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11
+
12
+
13
+def sample_requests(
14
+ dataset_path: str,
15
+ num_requests: int,
16
+ tokenizer: PreTrainedTokenizerBase,
17
+ fixed_output_len: Optional[int],
18
+) -> List[Tuple[str, int, int]]:
19
+ if fixed_output_len is not None and fixed_output_len < 4:
20
+ raise ValueError("output_len too small")
21
+
22
+ # Load the dataset.
23
+ with open(dataset_path) as f:
24
+ dataset = json.load(f)
25
+ # Filter out the conversations with less than 2 turns.
26
+ dataset = [data for data in dataset if len(data["conversations"]) >= 2]
27
+ # Only keep the first two turns of each conversation.
28
+ dataset = [(data["conversations"][0]["value"],
29
+ data["conversations"][1]["value"]) for data in dataset]
30
+
31
+ # Shuffle the dataset.
32
+ random.shuffle(dataset)
33
+
34
+ # Filter out sequences that are too long or too short
35
+ filtered_dataset: List[Tuple[str, int, int]] = []
36
+ for i in range(len(dataset)):
37
+ if len(filtered_dataset) == num_requests:
38
+ break
39
+
40
+ # Tokenize the prompts and completions.
41
+ prompt = dataset[i][0]
42
+ prompt_token_ids = tokenizer(prompt).input_ids
43
+ completion = dataset[i][1]
44
+ completion_token_ids = tokenizer(completion).input_ids
45
+ prompt_len = len(prompt_token_ids)
46
+ output_len = len(completion_token_ids
47
+ ) if fixed_output_len is None else fixed_output_len
48
+ if prompt_len < 4 or output_len < 4:
49
+ # Prune too short sequences.
50
+ continue
51
+ if prompt_len > 1024 or prompt_len + output_len > 2048:
52
+ # Prune too long sequences.
53
+ continue
54
+
55
+ #Select a equi-probable random priority
56
+ priority = 0 if random.random() < 0.5 else 1
57
+
58
+ filtered_dataset.append((prompt, prompt_len, output_len, priority))
59
+
60
+ return filtered_dataset
61
+
62
+
63
+def run_vllm(
64
+ requests: List[Tuple[str, int, int]],
65
+ model: str,
66
+ tokenizer: str,
67
+ quantization: Optional[str],
68
+ tensor_parallel_size: int,
69
+ seed: int,
70
+ n: int,
71
+ use_beam_search: bool,
72
+ trust_remote_code: bool,
73
+ dtype: str,
74
+ max_model_len: Optional[int],
75
+ enforce_eager: bool,
76
+ kv_cache_dtype: str,
77
+ quantization_param_path: Optional[str],
78
+ device: str,
79
+ enable_prefix_caching: bool,
80
+ enable_chunked_prefill: bool,
81
+ max_num_batched_tokens: int,
82
+ gpu_memory_utilization: float = 0.9,
83
+ download_dir: Optional[str] = None,
84
+) -> float:
85
+ from vllm import LLM, SamplingParams
86
+ llm = LLM(
87
+ model=model,
88
+ tokenizer=tokenizer,
89
+ quantization=quantization,
90
+ tensor_parallel_size=tensor_parallel_size,
91
+ seed=seed,
92
+ trust_remote_code=trust_remote_code,
93
+ dtype=dtype,
94
+ max_model_len=max_model_len,
95
+ gpu_memory_utilization=gpu_memory_utilization,
96
+ enforce_eager=enforce_eager,
97
+ kv_cache_dtype=kv_cache_dtype,
98
+ quantization_param_path=quantization_param_path,
99
+ device=device,
100
+ enable_prefix_caching=enable_prefix_caching,
101
+ download_dir=download_dir,
102
+ enable_chunked_prefill=enable_chunked_prefill,
103
+ max_num_batched_tokens=max_num_batched_tokens,
104
+ disable_log_stats=False,
105
+ )
106
+
107
+ # Add the requests to the engine.
108
+ prompts = []
109
+ sampling_params = []
110
+ priority = []
111
+ for prompt, _, output_len, _priority in requests:
112
+ prompts.append(prompt)
113
+ priority.append(_priority)
114
+ sampling_params.append(
115
+ SamplingParams(
116
+ n=n,
117
+ temperature=0.0 if use_beam_search else 1.0,
118
+ top_p=1.0,
119
+ use_beam_search=use_beam_search,
120
+ ignore_eos=True,
121
+ max_tokens=output_len,
122
+ ))
123
+
124
+ start = time.perf_counter()
125
+ llm.generate(prompts, sampling_params, priority=priority, use_tqdm=True)
126
+ end = time.perf_counter()
127
+ return end - start
128
+
129
+
130
+def main(args: argparse.Namespace):
131
+ print(args)
132
+ random.seed(args.seed)
133
+
134
+ # Sample the requests.
135
+ tokenizer = AutoTokenizer.from_pretrained(
136
+ args.tokenizer, trust_remote_code=args.trust_remote_code)
137
+ if args.dataset is None:
138
+ # Synthesize a prompt with the given input length.
139
+ prompt = "hi" * (args.input_len - 1)
140
+ requests = [(prompt, args.input_len, args.output_len)
141
+ for _ in range(args.num_prompts)]
142
+ else:
143
+ requests = sample_requests(args.dataset, args.num_prompts, tokenizer,
144
+ args.output_len)
145
+
146
+ if args.backend == "vllm":
147
+ elapsed_time = run_vllm(
148
+ requests, args.model, args.tokenizer, args.quantization,
149
+ args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
150
+ args.trust_remote_code, args.dtype, args.max_model_len,
151
+ args.enforce_eager, args.kv_cache_dtype,
152
+ args.quantization_param_path, args.device,
153
+ args.enable_prefix_caching, args.enable_chunked_prefill,
154
+ args.max_num_batched_tokens, args.gpu_memory_utilization,
155
+ args.download_dir)
156
+ else:
157
+ raise ValueError(f"Unknown backend: {args.backend}")
158
+ total_num_tokens = sum(prompt_len + output_len
159
+ for _, prompt_len, output_len, priority in requests)
160
+ print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
161
+ f"{total_num_tokens / elapsed_time:.2f} tokens/s")
162
+
163
+ # Output JSON results if specified
164
+ if args.output_json:
165
+ results = {
166
+ "elapsed_time": elapsed_time,
167
+ "num_requests": len(requests),
168
+ "total_num_tokens": total_num_tokens,
169
+ "requests_per_second": len(requests) / elapsed_time,
170
+ "tokens_per_second": total_num_tokens / elapsed_time,
171
+ }
172
+ with open(args.output_json, "w") as f:
173
+ json.dump(results, f, indent=4)
174
+
175
+
176
+if __name__ == "__main__":
177
+ parser = argparse.ArgumentParser(description="Benchmark the throughput.")
178
+ parser.add_argument("--backend",
179
+ type=str,
180
+ choices=["vllm", "hf", "mii"],
181
+ default="vllm")
182
+ parser.add_argument("--dataset",
183
+ type=str,
184
+ default=None,
185
+ help="Path to the dataset.")
186
+ parser.add_argument("--input-len",
187
+ type=int,
188
+ default=None,
189
+ help="Input prompt length for each request")
190
+ parser.add_argument("--output-len",
191
+ type=int,
192
+ default=None,
193
+ help="Output length for each request. Overrides the "
194
+ "output length from the dataset.")
195
+ parser.add_argument("--model", type=str, default="facebook/opt-125m")
196
+ parser.add_argument("--tokenizer", type=str, default=None)
197
+ parser.add_argument('--quantization',
198
+ '-q',
199
+ choices=[*QUANTIZATION_METHODS, None],
200
+ default=None)
201
+ parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1)
202
+ parser.add_argument("--n",
203
+ type=int,
204
+ default=1,
205
+ help="Number of generated sequences per prompt.")
206
+ parser.add_argument("--use-beam-search", action="store_true")
207
+ parser.add_argument("--num-prompts",
208
+ type=int,
209
+ default=200,
210
+ help="Number of prompts to process.")
211
+ parser.add_argument("--seed", type=int, default=0)
212
+ parser.add_argument('--trust-remote-code',
213
+ action='store_true',
214
+ help='trust remote code from huggingface')
215
+ parser.add_argument(
216
+ '--max-model-len',
217
+ type=int,
218
+ default=None,
219
+ help='Maximum length of a sequence (including prompt and output). '
220
+ 'If None, will be derived from the model.')
221
+ parser.add_argument(
222
+ '--dtype',
223
+ type=str,
224
+ default='auto',
225
+ choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'],
226
+ help='data type for model weights and activations. '
227
+ 'The "auto" option will use FP16 precision '
228
+ 'for FP32 and FP16 models, and BF16 precision '
229
+ 'for BF16 models.')
230
+ parser.add_argument('--gpu-memory-utilization',
231
+ type=float,
232
+ default=0.9,
233
+ help='the fraction of GPU memory to be used for '
234
+ 'the model executor, which can range from 0 to 1.'
235
+ 'If unspecified, will use the default value of 0.9.')
236
+ parser.add_argument("--enforce-eager",
237
+ action="store_true",
238
+ help="enforce eager execution")
239
+ parser.add_argument(
240
+ '--kv-cache-dtype',
241
+ type=str,
242
+ choices=['auto', 'fp8', 'fp8_e5m2', 'fp8_e4m3'],
243
+ default="auto",
244
+ help='Data type for kv cache storage. If "auto", will use model '
245
+ 'data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. '
246
+ 'ROCm (AMD GPU) supports fp8 (=fp8_e4m3)')
247
+ parser.add_argument(
248
+ '--quantization-param-path',
249
+ type=str,
250
+ default=None,
251
+ help='Path to the JSON file containing the KV cache scaling factors. '
252
+ 'This should generally be supplied, when KV cache dtype is FP8. '
253
+ 'Otherwise, KV cache scaling factors default to 1.0, which may cause '
254
+ 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
255
+ 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
256
+ 'instead supported for common inference criteria.')
257
+ parser.add_argument(
258
+ "--device",
259
+ type=str,
260
+ default="cuda",
261
+ choices=["cuda", "cpu"],
262
+ help='device type for vLLM execution, supporting CUDA and CPU.')
263
+ parser.add_argument(
264
+ "--enable-prefix-caching",
265
+ action='store_true',
266
+ help="enable automatic prefix caching for vLLM backend.")
267
+ parser.add_argument("--enable-chunked-prefill",
268
+ action='store_true',
269
+ help="enable chunked prefill for vLLM backend.")
270
+ parser.add_argument('--max-num-batched-tokens',
271
+ type=int,
272
+ default=None,
273
+ help='maximum number of batched tokens per '
274
+ 'iteration')
275
+ parser.add_argument('--download-dir',
276
+ type=str,
277
+ default=None,
278
+ help='directory to download and load the weights, '
279
+ 'default to the default cache dir of huggingface')
280
+ parser.add_argument(
281
+ '--output-json',
282
+ type=str,
283
+ default=None,
284
+ help='Path to save the throughput results in JSON format.')
285
+
286
+ args = parser.parse_args()
287
+ if args.tokenizer is None:
288
+ args.tokenizer = args.model
289
+ if args.dataset is None:
290
+ assert args.input_len is not None
291
+ assert args.output_len is not None
292
+ else:
293
+ assert args.input_len is None
294
+
295
+ main(args)
vllm/config.py
+4
-2
@@ -961,7 +961,7 @@ class SchedulerConfig:
961
workers instead of an entire data. It should be enabled only
962
when SPMD worker architecture is enabled. I.e.,
963
VLLM_USE_RAY_SPMD_WORKER=1
964
-
964
+ policy: The scheduling policy to use. "fcfs" (default) or "priority".
965
"""
966
967
def __init__(self,
@@ -977,7 +977,8 @@ class SchedulerConfig:
977
preemption_mode: Optional[str] = None,
978
num_scheduler_steps: int = 1,
979
multi_step_stream_outputs: bool = False,
980
- send_delta_data: bool = False) -> None:
980
+ send_delta_data: bool = False,
981
+ policy: str = "fcfs") -> None:
982
if max_num_batched_tokens is None:
983
if enable_chunked_prefill:
984
# It is the values that have the best balance between ITL
@@ -1019,6 +1020,7 @@ class SchedulerConfig:
1020
self.num_scheduler_steps = num_scheduler_steps
1021
self.multi_step_stream_outputs = multi_step_stream_outputs
1022
self.send_delta_data = send_delta_data
1023
+ self.policy = policy
1024
self._verify_args()
1025
1026
def _verify_args(self) -> None:
vllm/core/scheduler.py
+77
@@ -766,6 +766,79 @@ class Scheduler:
766
else:
767
return prompt_limit
768
769
+ def _get_priority(self,
770
+ seq_group: SequenceGroup) -> Tuple[Optional[int], float]:
771
+ """ Get the priority of the sequence group.
772
+ Highest preference to user-defined priority, followed by arrival time.
773
+ Args:
774
+ seq_group: The sequence group input.
775
+ Returns:
776
+ The priority of the sequence group.
777
+ """
778
+ return seq_group.priority, seq_group.arrival_time
779
+
780
+ def _schedule_priority_preemption(
781
+ self,
782
+ budget: SchedulingBudget,
783
+ ) -> int:
784
+ """Sorts waiting and running queue. Also, force preempt requests
785
+ from the running queue if their priority is lower.
786
+ Priority-based preemption is used with the priority policy.
787
+ Args:
788
+ budget: The scheduling budget. The argument is in-place updated
789
+ when any requests are scheduled.
790
+ Returns:
791
+ A count of priority-based preemptions.
792
+ """
793
+
794
+ waiting_queue = self.waiting
795
+
796
+ running_queue = deque(sorted(self.running, key=self._get_priority))
797
+
798
+ blocks_to_swap_out: List[Tuple[int, int]] = []
799
+ force_preemption_count = 0
800
+
801
+ if waiting_queue:
802
+ seq_group = waiting_queue.popleft()
803
+ num_new_seqs = seq_group.get_max_num_running_seqs()
804
+ num_new_tokens = self._get_num_new_tokens(seq_group,
805
+ SequenceStatus.WAITING,
806
+ False, budget)
807
+
808
+ #Only preempt if priority inversion exists
809
+ while running_queue and self._get_priority(
810
+ running_queue[-1]) > self._get_priority(seq_group):
811
+ #Only preempt if waiting sequence cannot be allocated
812
+ can_allocate = self.block_manager.can_allocate(seq_group)
813
+ if (num_new_tokens and can_allocate == AllocStatus.OK
814
+ and budget.can_schedule(num_new_tokens=num_new_tokens,
815
+ num_new_seqs=num_new_seqs)):
816
+ break
817
+
818
+ #Adjust budget to remove the victim sequence group
819
+ vseq_group = running_queue.pop()
820
+ num_running_tokens = self._get_num_new_tokens(
821
+ vseq_group, SequenceStatus.RUNNING, False, budget)
822
+ budget.subtract_num_batched_tokens(vseq_group.request_id,
823
+ num_running_tokens)
824
+ num_running_seqs = vseq_group.get_max_num_running_seqs()
825
+ budget.subtract_num_seqs(vseq_group.request_id,
826
+ num_running_seqs)
827
+
828
+ #Preempt out the victim sequence group
829
+ self._preempt(vseq_group, blocks_to_swap_out,
830
+ PreemptionMode.RECOMPUTE)
831
+ waiting_queue.appendleft(vseq_group)
832
+ force_preemption_count += 1
833
+ #Put the sequence back into the waiting queue
834
+ waiting_queue.appendleft(seq_group)
835
+
836
+ waiting_queue = deque(sorted(waiting_queue, key=self._get_priority))
837
+
838
+ self.waiting = waiting_queue
839
+ self.running = running_queue
840
+ return force_preemption_count
841
+
842
def _schedule_prefills(
843
self,
844
budget: SchedulingBudget,
@@ -917,6 +990,10 @@ class Scheduler:
990
curr_loras,
991
enable_chunking=False)
992
993
+ if len(prefills.seq_groups
994
+ ) == 0 and self.scheduler_config.policy == "priority":
995
+ self._schedule_priority_preemption(budget)
996
+
997
# Don't schedule decodes if prefills are scheduled.
998
# NOTE: If `_schedule_prefills` doesn't enable chunking, self.running
999
# only contains decode requests, not chunked prefills.
vllm/engine/llm_engine.py
+20
-4
@@ -631,6 +631,7 @@ class LLMEngine:
631
lora_request: Optional[LoRARequest],
632
prompt_adapter_request: Optional[PromptAdapterRequest],
633
trace_headers: Optional[Mapping[str, str]] = None,
634
+ priority: int = 0,
635
) -> None:
636
self._validate_model_inputs(processed_inputs)
637
# Create the sequences.
@@ -661,7 +662,8 @@ class LLMEngine:
662
lora_request=lora_request,
663
trace_headers=trace_headers,
664
prompt_adapter_request=prompt_adapter_request,
664
- encoder_seq=encoder_seq)
665
+ encoder_seq=encoder_seq,
666
+ priority=priority)
667
elif isinstance(params, PoolingParams):
668
seq_group = self._create_sequence_group_with_pooling(
669
request_id,
@@ -670,7 +672,8 @@ class LLMEngine:
672
arrival_time=arrival_time,
673
lora_request=lora_request,
674
prompt_adapter_request=prompt_adapter_request,
673
- encoder_seq=encoder_seq)
675
+ encoder_seq=encoder_seq,
676
+ priority=priority)
677
else:
678
raise ValueError(
679
"Either SamplingParams or PoolingParams must be provided.")
@@ -695,6 +698,7 @@ class LLMEngine:
698
lora_request: Optional[LoRARequest] = None,
699
trace_headers: Optional[Mapping[str, str]] = None,
700
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
701
+ priority: int = 0,
702
) -> None:
703
"""Add a request to the engine's request pool.
704
@@ -713,6 +717,8 @@ class LLMEngine:
717
arrival_time: The arrival time of the request. If None, we use
718
the current monotonic time.
719
trace_headers: OpenTelemetry trace headers.
720
+ priority: The priority of the request.
721
+ Only applicable with priority scheduling.
722
723
Details:
724
- Set arrival_time to the current time if it is None.
@@ -741,6 +747,11 @@ class LLMEngine:
747
if lora_request is not None and not self.lora_config:
748
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
749
"not enabled!")
750
+
751
+ if priority > 0 and not self.scheduler_config.policy == "priority":
752
+ raise ValueError(f"Got priority {priority} but "
753
+ "Priority scheduling is not enabled.")
754
+
755
if arrival_time is None:
756
arrival_time = time.time()
757
@@ -760,6 +771,7 @@ class LLMEngine:
771
lora_request=lora_request,
772
prompt_adapter_request=prompt_adapter_request,
773
trace_headers=trace_headers,
774
+ priority=priority,
775
)
776
777
def _create_sequence_group_with_sampling(
@@ -772,6 +784,7 @@ class LLMEngine:
784
trace_headers: Optional[Mapping[str, str]] = None,
785
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
786
encoder_seq: Optional[Sequence] = None,
787
+ priority: int = 0,
788
) -> SequenceGroup:
789
"""Creates a SequenceGroup with SamplingParams."""
790
max_logprobs = self.get_model_config().max_logprobs
@@ -798,7 +811,8 @@ class LLMEngine:
811
lora_request=lora_request,
812
trace_headers=trace_headers,
813
prompt_adapter_request=prompt_adapter_request,
801
- encoder_seq=encoder_seq)
814
+ encoder_seq=encoder_seq,
815
+ priority=priority)
816
817
return seq_group
818
@@ -811,6 +825,7 @@ class LLMEngine:
825
lora_request: Optional[LoRARequest],
826
prompt_adapter_request: Optional[PromptAdapterRequest],
827
encoder_seq: Optional[Sequence] = None,
828
+ priority: int = 0,
829
) -> SequenceGroup:
830
"""Creates a SequenceGroup with PoolingParams."""
831
# Defensive copy of PoolingParams, which are used by the pooler
@@ -823,7 +838,8 @@ class LLMEngine:
838
lora_request=lora_request,
839
pooling_params=pooling_params,
840
prompt_adapter_request=prompt_adapter_request,
826
- encoder_seq=encoder_seq)
841
+ encoder_seq=encoder_seq,
842
+ priority=priority)
843
return seq_group
844
845
def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
vllm/entrypoints/llm.py
+10
-2
@@ -320,7 +320,8 @@ class LLM:
320
lora_request: Optional[Union[List[LoRARequest], LoRARequest]] = None,
321
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
322
guided_options_request: Optional[Union[LLMGuidedOptions,
323
- GuidedDecodingRequest]] = None
323
+ GuidedDecodingRequest]] = None,
324
+ priority: Optional[List[int]] = None,
325
) -> List[RequestOutput]:
326
"""Generates the completions for the input prompts.
327
@@ -339,6 +340,8 @@ class LLM:
340
lora_request: LoRA request to use for generation, if any.
341
prompt_adapter_request: Prompt Adapter request to use for
342
generation, if any.
343
+ priority: The priority of the requests, if any.
344
+ Only applicable when priority scheduling policy is enabled.
345
346
Returns:
347
A list of ``RequestOutput`` objects containing the
@@ -379,7 +382,8 @@ class LLM:
382
params=sampling_params,
383
lora_request=lora_request,
384
prompt_adapter_request=prompt_adapter_request,
382
- guided_options=guided_options_request)
385
+ guided_options=guided_options_request,
386
+ priority=priority)
387
388
outputs = self._run_engine(use_tqdm=use_tqdm)
389
return LLMEngine.validate_outputs(outputs, RequestOutput)
@@ -782,6 +786,7 @@ class LLM:
786
lora_request: Optional[Union[Sequence[LoRARequest], LoRARequest]],
787
prompt_adapter_request: Optional[PromptAdapterRequest],
788
guided_options: Optional[GuidedDecodingRequest] = None,
789
+ priority: Optional[List[int]] = None,
790
) -> None:
791
if isinstance(inputs, (str, dict)):
792
# Convert a single prompt to a list.
@@ -811,6 +816,7 @@ class LLM:
816
lora_request=lora_request[i] if isinstance(
817
lora_request, Sequence) else lora_request,
818
prompt_adapter_request=prompt_adapter_request,
819
+ priority=priority[i] if priority else 0,
820
)
821
822
def _add_request(
@@ -819,6 +825,7 @@ class LLM:
825
params: Union[SamplingParams, PoolingParams],
826
lora_request: Optional[LoRARequest] = None,
827
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
828
+ priority: int = 0,
829
) -> None:
830
request_id = str(next(self.request_counter))
831
self.llm_engine.add_request(
@@ -827,6 +834,7 @@ class LLM:
834
params,
835
lora_request=lora_request,
836
prompt_adapter_request=prompt_adapter_request,
837
+ priority=priority,
838
)
839
840
def _add_guided_processor(
vllm/sequence.py
+4
@@ -646,6 +646,7 @@ class SequenceGroup:
646
unless you are working with an encoder/decoder model.
647
trace_headers: OpenTelemetry trace headers.
648
prompt_adapter_request: Prompt Adapter request.
649
+ priority: User-defined priority of the request.
650
"""
651
652
def __init__(
@@ -660,9 +661,11 @@ class SequenceGroup:
661
encoder_seq: Optional[Sequence] = None,
662
trace_headers: Optional[Mapping[str, str]] = None,
663
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
664
+ priority: int = 0,
665
) -> None:
666
self.request_id = request_id
667
self.seqs = seqs
668
+ self.arrival_time = arrival_time
669
self.is_single_seq = len(seqs) == 1
670
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
671
@@ -680,6 +683,7 @@ class SequenceGroup:
683
self.prompt_adapter_request = prompt_adapter_request
684
self.encoder_seq = encoder_seq
685
self.trace_headers = trace_headers
686
+ self.priority = priority
687
688
self.cached_request_output = None
689