@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf2f084d56
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Dynamic scheduler delay to improve ITL performance (#3279)

Co-authored-by: Jan van Lunteren <jvl@zurich.ibm.com>

Thomas Parnell committed Mar 22, 2024 at 20:28 UTC cf2f084d56a1293cb08da2393984cdc7685ac019
4 files changed +72 -2
tests/core/test_scheduler.py
+34
@@ -1,5 +1,6 @@
1 from typing import List
2 import pytest # noqa
3 +import time
4
5 from vllm.config import CacheConfig, SchedulerConfig
6 from vllm.core.scheduler import Scheduler
@@ -168,3 +169,36 @@ def test_scheduler_max_seqs():
169 # and one is prompting.
170 _, out = scheduler.schedule()
171 assert set(out.scheduled_seq_groups) == set([all_seq_groups[1]])
172 +
173 +
174 +def test_scheduler_delay_factor():
175 +
176 + block_size = 4
177 + scheduler_config = SchedulerConfig(100, 64, 16, delay_factor=0.5)
178 + cache_config = CacheConfig(block_size, 1.0, 1, "auto")
179 + cache_config.num_cpu_blocks = 8
180 + cache_config.num_gpu_blocks = 8
181 + scheduler = Scheduler(scheduler_config, cache_config, None)
182 +
183 + # schedule first prompt
184 + _, seq_group = create_dummy_prompt("0", prompt_length=block_size)
185 + scheduler.add_seq_group(seq_group)
186 + seq_group_meta, out = scheduler.schedule()
187 + assert out.prompt_run
188 + assert seq_group_meta[0].request_id == '0'
189 +
190 + # wait for a second before scheduling next prompt
191 + time.sleep(1)
192 + _, seq_group = create_dummy_prompt("1", prompt_length=block_size)
193 + scheduler.add_seq_group(seq_group)
194 +
195 + # second prompt should *not* be scheduled
196 + seq_group_meta, out = scheduler.schedule()
197 + assert not out.prompt_run
198 + assert seq_group_meta[0].request_id == '0'
199 +
200 + # wait for more than 0.5 second and try again
201 + time.sleep(0.6)
202 + seq_group_meta, out = scheduler.schedule()
203 + assert out.prompt_run
204 + assert seq_group_meta[0].request_id == '1'
vllm/config.py
+4
@@ -517,6 +517,8 @@ class SchedulerConfig:
517 iteration.
518 max_model_len: Maximum length of a sequence (including prompt
519 and generated text).
520 + delay_factor: Apply a delay (of delay factor multiplied by previous
521 + prompt latency) before scheduling next prompt.
522 """
523
524 def __init__(
@@ -524,6 +526,7 @@ class SchedulerConfig:
526 max_num_batched_tokens: Optional[int],
527 max_num_seqs: int,
528 max_model_len: int,
529 + delay_factor: float = 0.0,
530 ) -> None:
531 if max_num_batched_tokens is not None:
532 self.max_num_batched_tokens = max_num_batched_tokens
@@ -533,6 +536,7 @@ class SchedulerConfig:
536 self.max_num_batched_tokens = max(max_model_len, 2048)
537 self.max_num_seqs = max_num_seqs
538 self.max_model_len = max_model_len
539 + self.delay_factor = delay_factor
540 self._verify_args()
541
542 def _verify_args(self) -> None:
vllm/core/scheduler.py
+25 -1
@@ -103,6 +103,13 @@ class Scheduler:
103 # Sequence groups in the SWAPPED state.
104 self.swapped: Deque[SequenceGroup] = deque()
105
106 + # Time at previous scheduling step
107 + self.prev_time = 0.0
108 + # Did we schedule a prompt at previous step?
109 + self.prev_prompt = False
110 + # Latency of the last prompt step
111 + self.last_prompt_latency = 0.0
112 +
113 @property
114 def lora_enabled(self) -> bool:
115 return bool(self.lora_config)
@@ -179,7 +186,7 @@ class Scheduler:
186 # are added to the back.
187 leftover_waiting_sequences = deque()
188 num_batched_tokens = 0
182 - while self.waiting:
189 + while self._passed_delay(now) and self.waiting:
190 seq_group = self.waiting[0]
191 waiting_seqs = seq_group.get_seqs(
192 status=SequenceStatus.WAITING)
@@ -246,6 +253,7 @@ class Scheduler:
253 self.waiting.extendleft(leftover_waiting_sequences)
254
255 if scheduled or ignored_seq_groups:
256 + self.prev_prompt = True
257 scheduler_outputs = SchedulerOutputs(
258 scheduled_seq_groups=scheduled,
259 prompt_run=True,
@@ -491,3 +499,19 @@ class Scheduler:
499
500 def mark_blocks_as_computed(self, seq_group: SequenceGroup):
501 self.block_manager.mark_blocks_as_computed(seq_group)
502 +
503 + def _passed_delay(self, now: float) -> bool:
504 + if self.prev_prompt:
505 + self.last_prompt_latency = now - self.prev_time
506 + self.prev_time, self.prev_prompt = now, False
507 + # Delay scheduling prompts to let waiting queue fill up
508 + if self.scheduler_config.delay_factor > 0 and self.waiting:
509 + earliest_arrival_time = min(
510 + [e.metrics.arrival_time for e in self.waiting])
511 + passed_delay = (
512 + (now - earliest_arrival_time) >
513 + (self.scheduler_config.delay_factor * self.last_prompt_latency)
514 + or not self.running)
515 + else:
516 + passed_delay = True
517 + return passed_delay
vllm/engine/arg_utils.py
+9 -1
@@ -51,6 +51,7 @@ class EngineArgs:
51 max_cpu_loras: Optional[int] = None
52 device: str = 'auto'
53 ray_workers_use_nsight: bool = False
54 + scheduler_delay_factor: float = 0.0
55
56 def __post_init__(self):
57 if self.tokenizer is None:
@@ -305,6 +306,12 @@ class EngineArgs:
306 default=EngineArgs.device,
307 choices=["auto", "cuda", "neuron"],
308 help='Device type for vLLM execution.')
309 + parser.add_argument(
310 + '--scheduler-delay-factor',
311 + type=float,
312 + default=EngineArgs.scheduler_delay_factor,
313 + help='Apply a delay (of delay factor multiplied by previous'
314 + 'prompt latency) before scheduling next prompt.')
315 return parser
316
317 @classmethod
@@ -342,7 +349,8 @@ class EngineArgs:
349 ), self.ray_workers_use_nsight)
350 scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
351 self.max_num_seqs,
345 - model_config.max_model_len)
352 + model_config.max_model_len,
353 + self.scheduler_delay_factor)
354 lora_config = LoRAConfig(
355 max_lora_rank=self.max_lora_rank,
356 max_loras=self.max_loras,