A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Dynamic scheduler delay to improve ITL performance (#3279)
Co-authored-by: Jan van Lunteren <jvl@zurich.ibm.com>
Thomas Parnell committed
Mar 22, 2024 at 20:28 UTC
cf2f084d56a1293cb08da2393984cdc7685ac019
4 files changed
+72
-2
tests/core/test_scheduler.py
+34
@@ -1,5 +1,6 @@
1
from typing import List
2
import pytest # noqa
3
+import time
4
5
from vllm.config import CacheConfig, SchedulerConfig
6
from vllm.core.scheduler import Scheduler
@@ -168,3 +169,36 @@ def test_scheduler_max_seqs():
169
# and one is prompting.
170
_, out = scheduler.schedule()
171
assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
172
+
173
+
174
+def test_scheduler_delay_factor():
175
+
176
+ block_size = 4
177
+ scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
178
+ cache_config = CacheConfig(block_size, 1.0, 1, "auto")
179
+ cache_config.num_cpu_blocks = 8
180
+ cache_config.num_gpu_blocks = 8
181
+ scheduler = Scheduler(scheduler_config, cache_config, None)
182
+
183
+ # schedule first prompt
184
+ _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
185
+ scheduler.add_seq_group(seq_group)
186
+ seq_group_meta, out = scheduler.schedule()
187
+ assert out.prompt_run
188
+ assert seq_group_meta[0].request_id == '0'
189
+
190
+ # wait for a second before scheduling next prompt
191
+ time.sleep(1)
192
+ _, seq_group = create_dummy_prompt("1", prompt_length=block_size)
193
+ scheduler.add_seq_group(seq_group)
194
+
195
+ # second prompt should *not* be scheduled
196
+ seq_group_meta, out = scheduler.schedule()
197
+ assert not out.prompt_run
198
+ assert seq_group_meta[0].request_id == '0'
199
+
200
+ # wait for more than 0.5 second and try again
201
+ time.sleep(0.6)
202
+ seq_group_meta, out = scheduler.schedule()
203
+ assert out.prompt_run
204
+ assert seq_group_meta[0].request_id == '1'
vllm/config.py
+4
@@ -517,6 +517,8 @@ class SchedulerConfig:
517
iteration.
518
max_model_len: Maximum length of a sequence (including prompt
519
and generated text).
520
+ delay_factor: Apply a delay (of delay factor multiplied by previous
521
+ prompt latency) before scheduling next prompt.
522
"""
523
524
def __init__(
@@ -524,6 +526,7 @@ class SchedulerConfig:
526
max_num_batched_tokens: Optional[int],
527
max_num_seqs: int,
528
max_model_len: int,
529
+ delay_factor: float = 0.0,
530
) -> None:
531
if max_num_batched_tokens is not None:
532
self.max_num_batched_tokens = max_num_batched_tokens
@@ -533,6 +536,7 @@ class SchedulerConfig:
536
self.max_num_batched_tokens = max(max_model_len, 2048)
537
self.max_num_seqs = max_num_seqs
538
self.max_model_len = max_model_len
539
+ self.delay_factor = delay_factor
540
self._verify_args()
541
542
def _verify_args(self) -> None:
vllm/core/scheduler.py
+25
-1
@@ -103,6 +103,13 @@ class Scheduler:
103
# Sequence groups in the SWAPPED state.
104
self.swapped: Deque[SequenceGroup] = deque()
105
106
+ # Time at previous scheduling step
107
+ self.prev_time = 0.0
108
+ # Did we schedule a prompt at previous step?
109
+ self.prev_prompt = False
110
+ # Latency of the last prompt step
111
+ self.last_prompt_latency = 0.0
112
+
113
@property
114
def lora_enabled(self) -> bool:
115
return bool(self.lora_config)
@@ -179,7 +186,7 @@ class Scheduler:
186
# are added to the back.
187
leftover_waiting_sequences = deque()
188
num_batched_tokens = 0
182
- while self.waiting:
189
+ while self._passed_delay(now) and self.waiting:
190
seq_group = self.waiting[0]
191
waiting_seqs = seq_group.get_seqs(
192
status=SequenceStatus.WAITING)
@@ -246,6 +253,7 @@ class Scheduler:
253
self.waiting.extendleft(leftover_waiting_sequences)
254
255
if scheduled or ignored_seq_groups:
256
+ self.prev_prompt = True
257
scheduler_outputs = SchedulerOutputs(
258
scheduled_seq_groups=scheduled,
259
prompt_run=True,
@@ -491,3 +499,19 @@ class Scheduler:
499
500
def mark_blocks_as_computed(self, seq_group: SequenceGroup):
501
self.block_manager.mark_blocks_as_computed(seq_group)
502
+
503
+ def _passed_delay(self, now: float) -> bool:
504
+ if self.prev_prompt:
505
+ self.last_prompt_latency = now - self.prev_time
506
+ self.prev_time, self.prev_prompt = now, False
507
+ # Delay scheduling prompts to let waiting queue fill up
508
+ if self.scheduler_config.delay_factor > 0 and self.waiting:
509
+ earliest_arrival_time = min(
510
+ [e.metrics.arrival_time for e in self.waiting])
511
+ passed_delay = (
512
+ (now - earliest_arrival_time) >
513
+ (self.scheduler_config.delay_factor * self.last_prompt_latency)
514
+ or not self.running)
515
+ else:
516
+ passed_delay = True
517
+ return passed_delay
vllm/engine/arg_utils.py
+9
-1
@@ -51,6 +51,7 @@ class EngineArgs:
51
max_cpu_loras: Optional[int] = None
52
device: str = 'auto'
53
ray_workers_use_nsight: bool = False
54
+ scheduler_delay_factor: float = 0.0
55
56
def __post_init__(self):
57
if self.tokenizer is None:
@@ -305,6 +306,12 @@ class EngineArgs:
306
default=EngineArgs.device,
307
choices=["auto", "cuda", "neuron"],
308
help='Device type for vLLM execution.')
309
+ parser.add_argument(
310
+ '--scheduler-delay-factor',
311
+ type=float,
312
+ default=EngineArgs.scheduler_delay_factor,
313
+ help='Apply a delay (of delay factor multiplied by previous'
314
+ 'prompt latency) before scheduling next prompt.')
315
return parser
316
317
@classmethod
@@ -342,7 +349,8 @@ class EngineArgs:
349
), self.ray_workers_use_nsight)
350
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
351
self.max_num_seqs,
345
- model_config.max_model_len)
352
+ model_config.max_model_len,
353
+ self.scheduler_delay_factor)
354
lora_config = LoRAConfig(
355
max_lora_rank=self.max_lora_rank,
356
max_loras=self.max_loras,