@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55fe8a81ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Refactor scheduler (#658)

Woosuk Kwon committed Aug 2, 2023 at 16:42 UTC 55fe8a81ec5deab319c7c6b02913c21273b764ca
4 files changed +205 -144
examples/llm_engine_example.py
+1 -1
@@ -28,7 +28,7 @@ def main(args: argparse.Namespace):
28 # Run the engine by calling `engine.step()` manually.
29 request_id = 0
30 while True:
31 - # To test iteration-level scheduling, we add one request at each step.
31 + # To test continuous batching, we add one request at each step.
32 if test_prompts:
33 prompt, sampling_params = test_prompts.pop(0)
34 engine.add_request(str(request_id), prompt, sampling_params)
vllm/core/scheduler.py
+96 -129
@@ -12,8 +12,6 @@ from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
12
13 logger = init_logger(__name__)
14
15 -_LOGGING_INTERVAL_SEC = 5
16 -
15
16 class PreemptionMode(enum.Enum):
17 """Preemption modes.
@@ -32,19 +30,28 @@ class SchedulerOutputs:
30
31 def __init__(
32 self,
33 + scheduled_seq_groups: List[SequenceGroup],
34 + prompt_run: bool,
35 + num_batched_tokens: int,
36 blocks_to_swap_in: Dict[int, int],
37 blocks_to_swap_out: Dict[int, int],
38 blocks_to_copy: Dict[int, List[int]],
39 + ignored_seq_groups: List[SequenceGroup],
40 ) -> None:
41 + self.scheduled_seq_groups = scheduled_seq_groups
42 + self.prompt_run = prompt_run
43 + self.num_batched_tokens = num_batched_tokens
44 self.blocks_to_swap_in = blocks_to_swap_in
45 self.blocks_to_swap_out = blocks_to_swap_out
46 self.blocks_to_copy = blocks_to_copy
47 # Swap in and swap out should never happen at the same time.
48 assert not (blocks_to_swap_in and blocks_to_swap_out)
49 + self.ignored_seq_groups = ignored_seq_groups
50
51 def is_empty(self) -> bool:
46 - return (not self.blocks_to_swap_in and not self.blocks_to_swap_out
47 - and not self.blocks_to_copy)
52 + # NOTE: We do not consider the ignored sequence groups.
53 + return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
54 + and not self.blocks_to_swap_out and not self.blocks_to_copy)
55
56
57 class Scheduler:
@@ -53,11 +60,9 @@ class Scheduler:
60 self,
61 scheduler_config: SchedulerConfig,
62 cache_config: CacheConfig,
56 - log_stats: bool,
63 ) -> None:
64 self.scheduler_config = scheduler_config
65 self.cache_config = cache_config
60 - self.log_stats = log_stats
66
67 # Instantiate the scheduling policy.
68 self.policy = PolicyFactory.get_policy(policy_name="fcfs")
@@ -75,10 +80,6 @@ class Scheduler:
80 # Sequence groups in the SWAPPED state.
81 self.swapped: List[SequenceGroup] = []
82
78 - self.last_logging_time: float = 0.0
79 - # List[timestamp, num_tokens]
80 - self.num_input_tokens: List[Tuple[float, int]] = []
81 -
83 def add_seq_group(self, seq_group: SequenceGroup) -> None:
84 # Add sequence groups to the waiting queue.
85 self.waiting.append(seq_group)
@@ -101,21 +102,80 @@ class Scheduler:
102 def get_num_unfinished_seq_groups(self) -> int:
103 return len(self.waiting) + len(self.running) + len(self.swapped)
104
104 - def _schedule(
105 - self) -> Tuple[SchedulerOutputs, List[str], List[SequenceGroup]]:
105 + def _schedule(self) -> SchedulerOutputs:
106 # Blocks that need to be swaped or copied before model execution.
107 blocks_to_swap_in: Dict[int, int] = {}
108 blocks_to_swap_out: Dict[int, int] = {}
109 blocks_to_copy: Dict[int, List[int]] = {}
110 - ignored_seq_groups: List[SequenceGroup] = []
110
111 # Fix the current time.
112 now = time.time()
113
115 - # NOTE(woosuk): We prioritize the sequence groups in the RUNNING state
116 - # in order to minimize the preemption overheads.
117 - # Preemption happens only when there is no available slot to keep all
118 - # the sequence groups in the RUNNING state.
114 + # Join waiting sequences if possible.
115 + if not self.swapped:
116 + ignored_seq_groups: List[SequenceGroup] = []
117 + scheduled: List[SequenceGroup] = []
118 + num_batched_tokens = 0
119 + # Optimization: We do not sort the waiting queue since the preempted
120 + # sequence groups are added to the front and the new sequence groups
121 + # are added to the back.
122 + while self.waiting:
123 + seq_group = self.waiting[0]
124 +
125 + num_prompt_tokens = seq_group.get_seqs()[0].get_len()
126 + prompt_limit = min(
127 + self.scheduler_config.max_model_len,
128 + self.scheduler_config.max_num_batched_tokens)
129 + if num_prompt_tokens > prompt_limit:
130 + logger.warning(
131 + f"Input prompt ({num_prompt_tokens} tokens) is too long"
132 + f" and exceeds limit of {prompt_limit}")
133 + for seq in seq_group.get_seqs():
134 + seq.status = SequenceStatus.FINISHED_IGNORED
135 + ignored_seq_groups.append(seq_group)
136 + self.waiting.pop(0)
137 + break
138 +
139 + # If the sequence group cannot be allocated, stop.
140 + if not self.block_manager.can_allocate(seq_group):
141 + break
142 +
143 + # If the number of batched tokens exceeds the limit, stop.
144 + if (num_batched_tokens + num_prompt_tokens >
145 + self.scheduler_config.max_num_batched_tokens):
146 + break
147 +
148 + # The total number of sequences in the RUNNING state should not
149 + # exceed the maximum number of sequences.
150 + num_new_seqs = seq_group.num_seqs(
151 + status=SequenceStatus.WAITING)
152 + num_curr_seqs = sum(
153 + seq_group.num_seqs(status=SequenceStatus.RUNNING)
154 + for seq_group in self.running)
155 + if (num_curr_seqs + num_new_seqs >
156 + self.scheduler_config.max_num_seqs):
157 + break
158 +
159 + seq_group = self.waiting.pop(0)
160 + self._allocate(seq_group)
161 + self.running.append(seq_group)
162 + num_batched_tokens += num_prompt_tokens
163 + scheduled.append(seq_group)
164 +
165 + if scheduled:
166 + scheduler_outputs = SchedulerOutputs(
167 + scheduled_seq_groups=scheduled,
168 + prompt_run=True,
169 + num_batched_tokens=num_batched_tokens,
170 + blocks_to_swap_in=blocks_to_swap_in,
171 + blocks_to_swap_out=blocks_to_swap_out,
172 + blocks_to_copy=blocks_to_copy,
173 + ignored_seq_groups=ignored_seq_groups,
174 + )
175 + return scheduler_outputs
176 +
177 + # NOTE(woosuk): Preemption happens only when there is no available slot
178 + # to keep all the sequence groups in the RUNNING state.
179 # In this case, the policy is responsible for deciding which sequence
180 # groups to preempt.
181 self.running = self.policy.sort_by_priority(now, self.running)
@@ -173,124 +233,26 @@ class Scheduler:
233 seq_group.num_seqs(status=SequenceStatus.RUNNING)
234 for seq_group in self.running)
235
176 - # Join waiting sequences if possible.
177 - prompt_group_ids: List[str] = []
178 - # NOTE(woosuk): The sequence groups in the SWAPPED state are strictly
179 - # prioritized over the sequence groups in the WAITING state.
180 - # This is because we want to bound the amount of CPU memory taken by
181 - # the swapped sequence groups.
182 - if not self.swapped:
183 - # Optimization: We do not sort the waiting queue since the preempted
184 - # sequence groups are added to the front and the new sequence groups
185 - # are added to the back.
186 - while self.waiting:
187 - seq_group = self.waiting[0]
188 - # If the sequence group has been preempted in this step, stop.
189 - if seq_group in preempted:
190 - break
191 -
192 - num_prompt_tokens = seq_group.get_seqs()[0].get_len()
193 - prompt_limit = min(
194 - self.scheduler_config.max_model_len,
195 - self.scheduler_config.max_num_batched_tokens)
196 - if num_prompt_tokens > prompt_limit:
197 - logger.warning(
198 - f"Input prompt ({num_prompt_tokens} tokens) is too long"
199 - f" and exceeds limit of {prompt_limit}")
200 - for seq in seq_group.get_seqs():
201 - seq.status = SequenceStatus.FINISHED_IGNORED
202 - ignored_seq_groups.append(seq_group)
203 - self.waiting.pop(0)
204 - break
205 -
206 - # If the sequence group cannot be allocated, stop.
207 - if not self.block_manager.can_allocate(seq_group):
208 - break
209 -
210 - # If the number of batched tokens exceeds the limit, stop.
211 - if (num_batched_tokens + num_prompt_tokens >
212 - self.scheduler_config.max_num_batched_tokens):
213 - break
214 -
215 - # The total number of sequences in the RUNNING state should not
216 - # exceed the maximum number of sequences.
217 - num_new_seqs = seq_group.num_seqs(
218 - status=SequenceStatus.WAITING)
219 - num_curr_seqs = sum(
220 - seq_group.num_seqs(status=SequenceStatus.RUNNING)
221 - for seq_group in self.running)
222 - if (num_curr_seqs + num_new_seqs >
223 - self.scheduler_config.max_num_seqs):
224 - break
225 -
226 - seq_group = self.waiting.pop(0)
227 - self._allocate(seq_group)
228 - self.running.append(seq_group)
229 - num_batched_tokens += num_prompt_tokens
230 - prompt_group_ids.append(seq_group.request_id)
231 -
236 scheduler_outputs = SchedulerOutputs(
237 + scheduled_seq_groups=self.running,
238 + prompt_run=False,
239 + num_batched_tokens=num_batched_tokens,
240 blocks_to_swap_in=blocks_to_swap_in,
241 blocks_to_swap_out=blocks_to_swap_out,
242 blocks_to_copy=blocks_to_copy,
243 + ignored_seq_groups=[],
244 )
237 - if not self.log_stats:
238 - return scheduler_outputs, prompt_group_ids, ignored_seq_groups
245 + return scheduler_outputs
246
240 - # TODO(woosuk): Move the below code to the engine.
241 - now = time.time()
242 - if num_batched_tokens > 0:
243 - self.num_input_tokens.append((now, num_batched_tokens))
244 - elapsed_time = now - self.last_logging_time
245 - if elapsed_time > _LOGGING_INTERVAL_SEC:
246 - self.last_logging_time = now
247 - self.num_input_tokens = [(t, n) for t, n in self.num_input_tokens
248 - if now - t < _LOGGING_INTERVAL_SEC]
249 - if len(self.num_input_tokens) > 1:
250 - total_num_tokens = sum(n
251 - for _, n in self.num_input_tokens[:-1])
252 - window = now - self.num_input_tokens[0][0]
253 - avg_throughput = total_num_tokens / window
254 - else:
255 - avg_throughput = 0.0
256 -
257 - total_num_gpu_blocks = self.cache_config.num_gpu_blocks
258 - num_free_gpu_blocks = self.block_manager.get_num_free_gpu_blocks()
259 - num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
260 - gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
261 -
262 - total_num_cpu_blocks = self.cache_config.num_cpu_blocks
263 - if total_num_cpu_blocks > 0:
264 - num_free_cpu_blocks = (
265 - self.block_manager.get_num_free_cpu_blocks())
266 - num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
267 - cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
268 - else:
269 - cpu_cache_usage = 0.0
270 -
271 - logger.info(f"Throughput: {avg_throughput:.1f} tokens/s, "
272 - f"Running: {len(self.running)} reqs, "
273 - f"Swapped: {len(self.swapped)} reqs, "
274 - f"Pending: {len(self.waiting)} reqs, "
275 - f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
276 - f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
277 - return scheduler_outputs, prompt_group_ids, ignored_seq_groups
278 -
279 - def schedule(
280 - self
281 - ) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs,
282 - List[SequenceGroup]]:
247 + def schedule(self) -> Tuple[List[SequenceGroupMetadata], SchedulerOutputs]:
248 # Schedule sequence groups.
249 # This function call changes the internal states of the scheduler
250 # such as self.running, self.swapped, and self.waiting.
286 - (scheduler_outputs, prompt_group_ids,
287 - ignored_seq_groups) = self._schedule()
251 + scheduler_outputs = self._schedule()
252
253 # Create input data structures.
254 seq_group_metadata_list: List[SequenceGroupMetadata] = []
291 - for seq_group in self.running:
292 - is_prompt = seq_group.request_id in prompt_group_ids
293 -
255 + for seq_group in scheduler_outputs.scheduled_seq_groups:
256 seq_data: Dict[int, List[SequenceData]] = {}
257 block_tables: Dict[int, List[int]] = {}
258 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
@@ -300,20 +262,27 @@ class Scheduler:
262
263 seq_group_metadata = SequenceGroupMetadata(
264 request_id=seq_group.request_id,
303 - is_prompt=is_prompt,
265 + is_prompt=scheduler_outputs.prompt_run,
266 seq_data=seq_data,
267 sampling_params=seq_group.sampling_params,
268 block_tables=block_tables,
269 )
270 seq_group_metadata_list.append(seq_group_metadata)
309 - return seq_group_metadata_list, scheduler_outputs, ignored_seq_groups
271 + return seq_group_metadata_list, scheduler_outputs
272
273 def update(
274 self,
275 seq_outputs: Dict[int, SequenceOutputs],
276 ) -> List[SequenceGroup]:
315 - # Update the running sequences and free blocks.
277 + scheduled: List[SequenceGroup] = []
278 for seq_group in self.running:
279 + for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
280 + if seq.seq_id in seq_outputs:
281 + scheduled.append(seq_group)
282 + break
283 +
284 + # Update the scheduled sequences and free blocks.
285 + for seq_group in scheduled:
286 # Process beam search results before processing the new tokens.
287 for seq in seq_group.get_seqs(status=SequenceStatus.RUNNING):
288 output = seq_outputs[seq.seq_id]
@@ -331,9 +300,7 @@ class Scheduler:
300 # Append a new token to the sequence.
301 output = seq_outputs[seq.seq_id]
302 seq.append_token_id(output.output_token, output.logprobs)
334 - # Return a shallow copy of the running queue to prevent the queue
335 - # from being modified by the caller.
336 - return self.running.copy()
303 + return scheduled
304
305 def free_seq(self, seq: Sequence, finish_status: SequenceStatus) -> None:
306 seq.status = finish_status
vllm/engine/llm_engine.py
+91 -9
@@ -1,7 +1,7 @@
1 import time
2 import copy
3 from functools import partial
4 -from typing import Any, List, Optional, TYPE_CHECKING
4 +from typing import Any, List, Optional, Tuple, TYPE_CHECKING
5
6 from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
7 SchedulerConfig)
@@ -25,6 +25,8 @@ if TYPE_CHECKING:
25
26 logger = init_logger(__name__)
27
28 +_LOGGING_INTERVAL_SEC = 5
29 +
30
31 class LLMEngine:
32 """An LLM engine that receives requests and generates texts.
@@ -102,7 +104,14 @@ class LLMEngine:
104 self._init_cache()
105
106 # Create the scheduler.
105 - self.scheduler = Scheduler(scheduler_config, cache_config, log_stats)
107 + self.scheduler = Scheduler(scheduler_config, cache_config)
108 +
109 + # Logging.
110 + self.last_logging_time = 0.0
111 + # List of (timestamp, num_tokens)
112 + self.num_prompt_tokens: List[Tuple[float, int]] = []
113 + # List of (timestamp, num_tokens)
114 + self.num_generation_tokens: List[Tuple[float, int]] = []
115
116 def _init_workers(self, distributed_init_method: str):
117 # Lazy import the Worker to avoid importing torch.cuda/xformers
@@ -288,12 +297,17 @@ class LLMEngine:
297 and updates the scheduler with the model outputs. Finally, it decodes
298 the sequences and returns the newly generated results.
299 """
291 - (seq_group_metadata_list, scheduler_outputs,
292 - ignored_seq_groups) = self.scheduler.schedule()
293 - if ((not seq_group_metadata_list) and scheduler_outputs.is_empty()
294 - and (not ignored_seq_groups)):
295 - # Nothing to do.
296 - return []
300 + seq_group_metadata_list, scheduler_outputs = self.scheduler.schedule()
301 + if scheduler_outputs.is_empty():
302 + if not scheduler_outputs.ignored_seq_groups:
303 + # Nothing to do.
304 + return []
305 + # If there are ignored seq groups, we need to return them as the
306 + # request outputs.
307 + return [
308 + RequestOutput.from_seq_group(seq_group)
309 + for seq_group in scheduler_outputs.ignored_seq_groups
310 + ]
311
312 # Execute the model.
313 output = self._run_workers(
@@ -315,11 +329,79 @@ class LLMEngine:
329
330 # Create the outputs.
331 request_outputs: List[RequestOutput] = []
318 - for seq_group in seq_groups + ignored_seq_groups:
332 + for seq_group in seq_groups + scheduler_outputs.ignored_seq_groups:
333 request_output = RequestOutput.from_seq_group(seq_group)
334 request_outputs.append(request_output)
335 +
336 + if self.log_stats:
337 + # Log the system stats.
338 + self._log_system_stats(scheduler_outputs.prompt_run,
339 + scheduler_outputs.num_batched_tokens)
340 return request_outputs
341
342 + def _log_system_stats(
343 + self,
344 + prompt_run: bool,
345 + num_batched_tokens: int,
346 + ) -> None:
347 + now = time.time()
348 + # Log the number of batched input tokens.
349 + if prompt_run:
350 + self.num_prompt_tokens.append((now, num_batched_tokens))
351 + else:
352 + self.num_generation_tokens.append((now, num_batched_tokens))
353 +
354 + elapsed_time = now - self.last_logging_time
355 + if elapsed_time < _LOGGING_INTERVAL_SEC:
356 + return
357 +
358 + # Discard the old stats.
359 + self.num_prompt_tokens = [(t, n) for t, n in self.num_prompt_tokens
360 + if now - t < _LOGGING_INTERVAL_SEC]
361 + self.num_generation_tokens = [(t, n)
362 + for t, n in self.num_generation_tokens
363 + if now - t < _LOGGING_INTERVAL_SEC]
364 +
365 + if len(self.num_prompt_tokens) > 1:
366 + total_num_tokens = sum(n for _, n in self.num_prompt_tokens[:-1])
367 + window = now - self.num_prompt_tokens[0][0]
368 + avg_prompt_throughput = total_num_tokens / window
369 + else:
370 + avg_prompt_throughput = 0.0
371 + if len(self.num_generation_tokens) > 1:
372 + total_num_tokens = sum(n
373 + for _, n in self.num_generation_tokens[:-1])
374 + window = now - self.num_generation_tokens[0][0]
375 + avg_generation_throughput = total_num_tokens / window
376 + else:
377 + avg_generation_throughput = 0.0
378 +
379 + total_num_gpu_blocks = self.cache_config.num_gpu_blocks
380 + num_free_gpu_blocks = (
381 + self.scheduler.block_manager.get_num_free_gpu_blocks())
382 + num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
383 + gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
384 +
385 + total_num_cpu_blocks = self.cache_config.num_cpu_blocks
386 + if total_num_cpu_blocks > 0:
387 + num_free_cpu_blocks = (
388 + self.scheduler.block_manager.get_num_free_cpu_blocks())
389 + num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
390 + cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
391 + else:
392 + cpu_cache_usage = 0.0
393 +
394 + logger.info("Avg prompt throughput: "
395 + f"{avg_prompt_throughput:.1f} tokens/s, "
396 + "Avg generation throughput: "
397 + f"{avg_generation_throughput:.1f} tokens/s, "
398 + f"Running: {len(self.scheduler.running)} reqs, "
399 + f"Swapped: {len(self.scheduler.swapped)} reqs, "
400 + f"Pending: {len(self.scheduler.waiting)} reqs, "
401 + f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
402 + f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
403 + self.last_logging_time = now
404 +
405 def _decode_sequences(self, seq_groups: List[SequenceGroup]) -> None:
406 """Decodes the sequence outputs."""
407 for seq_group in seq_groups:
vllm/model_executor/layers/attention.py
+17 -5
@@ -20,12 +20,20 @@ class PagedAttention(nn.Module):
20 """GPT-style multi-head PagedAttention.
21
22 This class takes flattened 1D query, key, and value tensors as input. The
23 - input 1D tensors can be split into three parts: the prompt tokens, the
24 - generation tokens, and the paddings.
23 + input 1D tensors can either contain prompt tokens or generation tokens, in
24 + addition to paddings.
25
26 - |<------------------------------------- num_valid_tokens ------------------------------------->|
27 - |<--------------- num_prompt_tokens -------------->|<------- num_generation_tokens (M) ------->|
28 - |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
26 + If the input tensors contain prompt tokens, the layout is as follows:
27 +
28 + |<---------------------- num_valid_tokens ---------------------->|
29 + |<--------------- num_prompt_tokens -------------->|
30 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|<--padding-->|
31 +
32 + Otherwise, the layout is as follows:
33 +
34 + |<------------------ num_valid_tokens ------------------->|
35 + |<------- num_generation_tokens (M) ------->|
36 + |<--generation_0-->|...|<--generation_M-1-->|<--padding-->|
37
38 The prompts might have different lengths, while the generation tokens always
39 have length 1. The paddings are appended to make the input length a multiple
@@ -188,6 +196,8 @@ class PagedAttention(nn.Module):
196 # Compute the attention op for prompts.
197 num_prompt_tokens = input_metadata.num_prompt_tokens
198 if num_prompt_tokens > 0:
199 + # Prompt run.
200 + assert input_metadata.num_generation_tokens == 0
201 self.set_attn_bias(input_metadata)
202 self.multi_query_kv_attention(
203 output[:num_prompt_tokens],
@@ -217,6 +227,8 @@ class PagedAttention(nn.Module):
227 )
228
229 if input_metadata.num_generation_tokens > 0:
230 + # Decoding run.
231 + assert input_metadata.num_prompt_tokens == 0
232 assert key_cache is not None and value_cache is not None, (
233 "key_cache and value_cache must be provided when "
234 "generating tokens.")